big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11315. [Federation] YARN Federation Router Supports Cross-Origin. (#4934)

Browse Source
This commit is contained in:
slfan1989 2022-10-12 01:18:50 +08:00 committed by GitHub
parent 2122733c30
commit 82a88a8ae6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 159 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -4194,6 +4194,11 @@ public static boolean isAclEnabled(Configuration conf) {
ROUTER_WEBAPP_PREFIX + "appsinfo-cached-count";
public static final int DEFAULT_ROUTER_APPSINFO_CACHED_COUNT = 100;
/** Enable cross origin (CORS) support. **/
public static final String ROUTER_WEBAPP_ENABLE_CORS_FILTER =
ROUTER_PREFIX + "webapp.cross-origin.enabled";
public static final boolean DEFAULT_ROUTER_WEBAPP_ENABLE_CORS_FILTER = false;
////////////////////////////////
// CSI Volume configs
////////////////////////////////

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
View File

@ -28,6 +28,7 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.util.Lists;
@ -299,4 +300,8 @@ static String getPrefix(String pathSpec) {
public abstract void setup();
@VisibleForTesting
public HttpServer2 getHttpServer() {
return httpServer;
}
}

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -5006,7 +5006,18 @@
Default is 100
</description>
</property>
<property>
<name>yarn.router.webapp.cross-origin.enabled</name>
<value>false</value>
<description>
Flag to enable cross-origin (CORS) support for Yarn Router.
For Yarn Router, also add
org.apache.hadoop.security.HttpCrossOriginFilterInitializer to the
configuration hadoop.http.filter.initializers in core-site.xml.
</description>
</property>
<property>
<name>yarn.federation.state-store.max-applications</name>
<value>1000</value>

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
View File

@ -129,6 +129,12 @@
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.glassfish.grizzly</groupId>
<artifactId>grizzly-http-servlet</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
View File

@ -27,6 +27,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.JvmPauseMonitor;
@ -168,6 +169,16 @@ public WebApp getWebapp() {
@VisibleForTesting
public void startWepApp() {
// Initialize RouterWeb's CrossOrigin capability.
boolean enableCors = conf.getBoolean(YarnConfiguration.ROUTER_WEBAPP_ENABLE_CORS_FILTER,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_ENABLE_CORS_FILTER);
if (enableCors) {
conf.setBoolean(HttpCrossOriginFilterInitializer.PREFIX
+ HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true);
}
LOG.info("Instantiating RouterWebApp at {}.", webAppAddress);
RMWebAppUtil.setupSecurityAndFilters(conf, null);
Builder<Object> builder =

109
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
View File

@ -22,11 +22,27 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
import org.apache.hadoop.security.http.CrossOriginFilter;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.webapp.WebAppContext;
import org.glassfish.grizzly.servlet.HttpServletResponseImpl;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
/**
* Tests {@link Router}.
@ -87,4 +103,97 @@ private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager,
}
}
@Test
public void testRouterSupportCrossOrigin() throws ServletException, IOException {
// We design test cases like this
// We start the Router and enable the Router to support Cross-origin.
// In the configuration, we allow example.com to access.
// 1. We simulate example.com and get the correct response
// 2. We simulate example.org and cannot get a response
// Initialize RouterWeb's CrossOrigin capability
Configuration conf = new Configuration();
conf.setBoolean(YarnConfiguration.ROUTER_WEBAPP_ENABLE_CORS_FILTER, true);
conf.set("hadoop.http.filter.initializers", HttpCrossOriginFilterInitializer.class.getName());
conf.set(HttpCrossOriginFilterInitializer.PREFIX + CrossOriginFilter.ALLOWED_ORIGINS,
"example.com");
conf.set(HttpCrossOriginFilterInitializer.PREFIX + CrossOriginFilter.ALLOWED_HEADERS,
"X-Requested-With,Accept");
conf.set(HttpCrossOriginFilterInitializer.PREFIX + CrossOriginFilter.ALLOWED_METHODS,
"GET,POST");
// Start the router
Router router = new Router();
router.init(conf);
router.start();
router.getServices();
// Get assigned to Filter.
// The name of the filter is "Cross Origin Filter",
// which is specified in HttpCrossOriginFilterInitializer.
WebApp webApp = router.getWebapp();
HttpServer2 httpServer2 = webApp.getHttpServer();
WebAppContext webAppContext = httpServer2.getWebAppContext();
ServletHandler servletHandler = webAppContext.getServletHandler();
FilterHolder holder = servletHandler.getFilter("Cross Origin Filter");
CrossOriginFilter filter = CrossOriginFilter.class.cast(holder.getFilter());
// 1. Simulate [example.com] for access
HttpServletRequest mockReq = Mockito.mock(HttpServletRequest.class);
Mockito.when(mockReq.getHeader("Origin")).thenReturn("example.com");
Mockito.when(mockReq.getHeader("Access-Control-Request-Method")).thenReturn("GET");
Mockito.when(mockReq.getHeader("Access-Control-Request-Headers"))
.thenReturn("X-Requested-With");
// Objects to verify interactions based on request
HttpServletResponseForRouterTest mockRes = new HttpServletResponseForRouterTest();
FilterChain mockChain = Mockito.mock(FilterChain.class);
// Object under test
filter.doFilter(mockReq, mockRes, mockChain);
// Why is 5, because when Filter passes,
// CrossOriginFilter will set 5 values to Map
Assert.assertEquals(5, mockRes.getHeaders().size());
String allowResult = mockRes.getHeader("Access-Control-Allow-Credentials");
Assert.assertEquals("true", allowResult);
// 2. Simulate [example.org] for access
HttpServletRequest mockReq2 = Mockito.mock(HttpServletRequest.class);
Mockito.when(mockReq2.getHeader("Origin")).thenReturn("example.org");
Mockito.when(mockReq2.getHeader("Access-Control-Request-Method")).thenReturn("GET");
Mockito.when(mockReq2.getHeader("Access-Control-Request-Headers"))
.thenReturn("X-Requested-With");
// Objects to verify interactions based on request
HttpServletResponseForRouterTest mockRes2 = new HttpServletResponseForRouterTest();
FilterChain mockChain2 = Mockito.mock(FilterChain.class);
// Object under test
filter.doFilter(mockReq2, mockRes2, mockChain2);
// Why is 0, because when the Filter fails,
// CrossOriginFilter will not set any value
Assert.assertEquals(0, mockRes2.getHeaders().size());
router.stop();
}
private class HttpServletResponseForRouterTest extends HttpServletResponseImpl {
private final Map<String, String> headers = new HashMap<>(1);
@Override
public void setHeader(String name, String value) {
headers.put(name, value);
}
public String getHeader(String name) {
return headers.get(name);
}
public Map<String, String> getHeaders() {
return headers;
}
}
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
View File

@ -284,6 +284,16 @@ Kerberos supported in federation.
| `yarn.router.kerberos.principal` | | The Router service principal. This is typically set to router/_HOST@REALM.TLD. Each Router will substitute _HOST with its own fully qualified hostname at startup. The _HOST placeholder allows using the same configuration setting on all Routers in setup. |
| `yarn.router.kerberos.principal.hostname` | | Optional. The hostname for the Router containing this configuration file. Will be different for each machine. Defaults to current hostname. |
Enabling CORS support:
To enable cross-origin support (CORS) for the Yarn Router, please set the following configuration parameters:
| Property | Example | Description |
| ----------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
| `hadoop.http.filter.initializers` | `org.apache.hadoop.security.HttpCrossOriginFilterInitializer` | Optional. Set the filter to HttpCrossOriginFilterInitializer, Configure this parameter in core-site.xml. |
| `yarn.router.webapp.cross-origin.enabled` | `true` | Optional. Enable/disable CORS filter.Configure this parameter in yarn-site.xml. |
###ON NMs:
These are extra configurations that should appear in the **conf/yarn-site.xml** at each NodeManager.

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md
View File

@ -41,6 +41,7 @@ origin (CORS) support.
| `yarn.timeline-service.http-cross-origin.enabled` | true | Enable CORS support for Timeline Server |
| `yarn.resourcemanager.webapp.cross-origin.enabled` | true | Enable CORS support for Resource Manager |
| `yarn.nodemanager.webapp.cross-origin.enabled` | true | Enable CORS support for Node Manager |
| `yarn.router.webapp.cross-origin.enabled` | true | Enable CORS support for Yarn Router |
Also please ensure that CORS related configurations are enabled in `core-site.xml`.
Kindly refer [here](../../hadoop-project-dist/hadoop-common/HttpAuthentication.html)