big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15707. Add IsActiveServlet to be used for Load Balancers. Contributed by Lukas Majercak.

Browse Source
This commit is contained in:
Giovanni Matteo Fumarola 2018-09-05 10:50:25 -07:00
parent e780556ae9
commit 9af96d4ed4
12 changed files with 307 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/IsActiveServlet.java Normal file
View File

@ -0,0 +1,71 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.http;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
/**
* Used by Load Balancers to detect the active NameNode/ResourceManager/Router.
*/
public abstract class IsActiveServlet extends HttpServlet {
/** Default serial identifier. */
private static final long serialVersionUID = 1L;
public static final String SERVLET_NAME = "isActive";
public static final String PATH_SPEC = "/isActive";
public static final String RESPONSE_ACTIVE =
"I am Active!";
public static final String RESPONSE_NOT_ACTIVE =
"I am not Active!";
/**
* Check whether this instance is the Active one.
* @param req HTTP request
* @param resp HTTP response to write to
*/
@Override
public void doGet(
final HttpServletRequest req, final HttpServletResponse resp)
throws IOException {
// By default requests are persistent. We don't want long-lived connections
// on server side.
resp.addHeader("Connection", "close");
if (!isActive()) {
// Report not SC_OK
resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED,
RESPONSE_NOT_ACTIVE);
return;
}
resp.setStatus(HttpServletResponse.SC_OK);
resp.getWriter().write(RESPONSE_ACTIVE);
resp.getWriter().flush();
}
/**
* @return true if this instance is in Active HA state.
*/
protected abstract boolean isActive();
}

95
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestIsActiveServlet.java Normal file
View File

@ -0,0 +1,95 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.http;
import org.junit.Before;
import org.junit.Test;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Test if the {@link IsActiveServlet} returns the right answer if the
* underlying service is active.
*/
public class TestIsActiveServlet {
private IsActiveServlet servlet;
private HttpServletRequest req;
private HttpServletResponse resp;
private ByteArrayOutputStream respOut;
@Before
public void setUp() throws Exception {
req = mock(HttpServletRequest.class);
resp = mock(HttpServletResponse.class);
respOut = new ByteArrayOutputStream();
PrintWriter writer = new PrintWriter(respOut);
when(resp.getWriter()).thenReturn(writer);
}
@Test
public void testSucceedsOnActive() throws IOException {
servlet = new IsActiveServlet() {
@Override
protected boolean isActive() {
return true;
}
};
String response = doGet();
verify(resp, never()).sendError(anyInt(), anyString());
assertEquals(IsActiveServlet.RESPONSE_ACTIVE, response);
}
@Test
public void testFailsOnInactive() throws IOException {
servlet = new IsActiveServlet() {
@Override
protected boolean isActive() {
return false;
}
};
doGet();
verify(resp, atLeastOnce()).sendError(
eq(HttpServletResponse.SC_METHOD_NOT_ALLOWED),
eq(IsActiveServlet.RESPONSE_NOT_ACTIVE));
}
private String doGet() throws IOException {
servlet.doGet(req, resp);
return new String(respOut.toByteArray(), "UTF-8");
}
}

37
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/IsRouterActiveServlet.java Normal file
View File

@ -0,0 +1,37 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.federation.router;
import org.apache.hadoop.http.IsActiveServlet;
import javax.servlet.ServletContext;
/**
* Detect if the Router is active and ready to serve requests.
*/
public class IsRouterActiveServlet extends IsActiveServlet {
@Override
protected boolean isActive() {
final ServletContext context = getServletContext();
final Router router = RouterHttpServer.getRouterFromContext(context);
final RouterServiceState routerState = router.getRouterState();
return routerState == RouterServiceState.RUNNING;
}
}

9
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
View File

@ -27,6 +27,8 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.service.AbstractService;
import javax.servlet.ServletContext;
/**
* Web interface for the {@link Router}. It exposes the Web UI and the WebHDFS
* methods from {@link RouterWebHdfsMethods}.
@ -116,6 +118,9 @@ public class RouterHttpServer extends AbstractService {
private static void setupServlets(
HttpServer2 httpServer, Configuration conf) {
// TODO Add servlets for FSCK, etc
httpServer.addInternalServlet(IsRouterActiveServlet.SERVLET_NAME,
IsRouterActiveServlet.PATH_SPEC,
IsRouterActiveServlet.class);
}
public InetSocketAddress getHttpAddress() {
@ -125,4 +130,8 @@ public class RouterHttpServer extends AbstractService {
public InetSocketAddress getHttpsAddress() {
return this.httpsAddress;
}
public static Router getRouterFromContext(ServletContext context) {
return (Router)context.getAttribute(NAMENODE_ATTRIBUTE_KEY);
}
}

2
hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
View File

@ -62,7 +62,7 @@ Each Router has two roles:
#### Federated interface
The Router receives a client request, checks the State Store for the correct subcluster, and forwards the request to the active NameNode of that subcluster.
The reply from the NameNode then flows in the opposite direction.
The Routers are stateless and can be behind a load balancer.
The Routers are stateless and can be behind a load balancer. For health checking, you can use /isActive endpoint as a health probe (e.g. http://ROUTER_HOSTNAME:ROUTER_PORT/isActive).
For performance, the Router also caches remote mount table entries and the state of the subclusters.
To make sure that changes have been propagated to all Routers, each Router heartbeats its state to the State Store.

33
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/IsNameNodeActiveServlet.java Normal file
View File

@ -0,0 +1,33 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.http.IsActiveServlet;
/**
* Used by Load Balancers to find the active NameNode.
*/
public class IsNameNodeActiveServlet extends IsActiveServlet {
@Override
protected boolean isActive() {
NameNode namenode = NameNodeHttpServer.getNameNodeFromContext(
getServletContext());
return namenode.isActiveState();
}
}

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
View File

@ -294,6 +294,9 @@ public class NameNodeHttpServer {
true);
httpServer.addInternalServlet("imagetransfer", ImageServlet.PATH_SPEC,
ImageServlet.class, true);
httpServer.addInternalServlet(IsNameNodeActiveServlet.SERVLET_NAME,
IsNameNodeActiveServlet.PATH_SPEC,
IsNameNodeActiveServlet.class);
}
static FSImage getFsImageFromContext(ServletContext context) {

8
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
View File

@ -423,6 +423,14 @@ This guide describes high-level uses of each of these subcommands. For specific
**Note:** This is not yet implemented, and at present will always return
success, unless the given NameNode is completely down.
### Load Balancer Setup
If you are running a set of NameNodes behind a Load Balancer (e.g. [Azure](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-custom-probe-overview) or [AWS](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-healthchecks.html) ) and would like the Load Balancer to point to the active NN, you can use the /isActive HTTP endpoint as a health probe.
http://NN_HOSTNAME/isActive will return a 200 status code response if the NN is in Active HA State, 405 otherwise.
Automatic Failover
------------------

38
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/IsResourceManagerActiveServlet.java Normal file
View File

@ -0,0 +1,38 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager;
import org.apache.hadoop.http.IsActiveServlet;
import org.apache.hadoop.ha.HAServiceProtocol;
/**
* Used by Load Balancers to find the active ResourceManager.
*/
public class IsResourceManagerActiveServlet extends IsActiveServlet {
public static final String RM_ATTRIBUTE = "rm";
@Override
protected boolean isActive() {
ResourceManager rm = (ResourceManager)
getServletContext().getAttribute(RM_ATTRIBUTE);
RMContext rmContext = rm.getRMContext();
HAServiceProtocol.HAServiceState state = rmContext.getHAServiceState();
return state == HAServiceProtocol.HAServiceState.ACTIVE;
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
View File

@ -1206,6 +1206,11 @@ public class ResourceManager extends CompositeService
}
}
builder.withAttribute(IsResourceManagerActiveServlet.RM_ATTRIBUTE, this);
builder.withServlet(IsResourceManagerActiveServlet.SERVLET_NAME,
IsResourceManagerActiveServlet.PATH_SPEC,
IsResourceManagerActiveServlet.class);
webApp = builder.start(new RMWebApp(this), uiWebAppContext);
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
View File

@ -37,6 +37,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HtmlQuoting;
import org.apache.hadoop.http.IsActiveServlet;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@ -68,7 +69,7 @@ public class RMWebAppFilter extends GuiceContainer {
// define a set of URIs which do not need to do redirection
private static final Set<String> NON_REDIRECTED_URIS = Sets.newHashSet(
"/conf", "/stacks", "/logLevel", "/logs");
"/conf", "/stacks", "/logLevel", "/logs", IsActiveServlet.PATH_SPEC);
private String path;
private boolean ahsEnabled;
private String ahsPageURLPrefix;

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md
View File

@ -144,3 +144,8 @@ Assuming a standby RM is up and running, the Standby automatically redirects all
### Web Services
Assuming a standby RM is up and running, RM web-services described at [ResourceManager REST APIs](./ResourceManagerRest.html) when invoked on a standby RM are automatically redirected to the Active RM.
### Load Balancer Setup
If you are running a set of ResourceManagers behind a Load Balancer (e.g. [Azure](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-custom-probe-overview) or [AWS](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-healthchecks.html) ) and would like the Load Balancer to point to the active RM, you can use the /isActive HTTP endpoint as a health probe.
http://RM_HOSTNAME/isActive will return a 200 status code response if the RM is in Active HA State, 405 otherwise.