diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/IsActiveServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/IsActiveServlet.java new file mode 100644 index 0000000000..3838beb199 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/IsActiveServlet.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.http; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * Used by Load Balancers to detect the active NameNode/ResourceManager/Router. + */ +public abstract class IsActiveServlet extends HttpServlet { + + /** Default serial identifier. */ + private static final long serialVersionUID = 1L; + + public static final String SERVLET_NAME = "isActive"; + public static final String PATH_SPEC = "/isActive"; + + public static final String RESPONSE_ACTIVE = + "I am Active!"; + + public static final String RESPONSE_NOT_ACTIVE = + "I am not Active!"; + + /** + * Check whether this instance is the Active one. + * @param req HTTP request + * @param resp HTTP response to write to + */ + @Override + public void doGet( + final HttpServletRequest req, final HttpServletResponse resp) + throws IOException { + + // By default requests are persistent. We don't want long-lived connections + // on server side. + resp.addHeader("Connection", "close"); + + if (!isActive()) { + // Report not SC_OK + resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED, + RESPONSE_NOT_ACTIVE); + return; + } + resp.setStatus(HttpServletResponse.SC_OK); + resp.getWriter().write(RESPONSE_ACTIVE); + resp.getWriter().flush(); + } + + /** + * @return true if this instance is in Active HA state. + */ + protected abstract boolean isActive(); +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestIsActiveServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestIsActiveServlet.java new file mode 100644 index 0000000000..5f5d51e1d7 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestIsActiveServlet.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.http; + + +import org.junit.Before; +import org.junit.Test; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintWriter; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + + +/** + * Test if the {@link IsActiveServlet} returns the right answer if the + * underlying service is active. + */ +public class TestIsActiveServlet { + + private IsActiveServlet servlet; + private HttpServletRequest req; + private HttpServletResponse resp; + private ByteArrayOutputStream respOut; + + @Before + public void setUp() throws Exception { + req = mock(HttpServletRequest.class); + resp = mock(HttpServletResponse.class); + respOut = new ByteArrayOutputStream(); + PrintWriter writer = new PrintWriter(respOut); + when(resp.getWriter()).thenReturn(writer); + } + + @Test + public void testSucceedsOnActive() throws IOException { + servlet = new IsActiveServlet() { + @Override + protected boolean isActive() { + return true; + } + }; + + String response = doGet(); + verify(resp, never()).sendError(anyInt(), anyString()); + assertEquals(IsActiveServlet.RESPONSE_ACTIVE, response); + } + + @Test + public void testFailsOnInactive() throws IOException { + servlet = new IsActiveServlet() { + @Override + protected boolean isActive() { + return false; + } + }; + + doGet(); + verify(resp, atLeastOnce()).sendError( + eq(HttpServletResponse.SC_METHOD_NOT_ALLOWED), + eq(IsActiveServlet.RESPONSE_NOT_ACTIVE)); + } + + private String doGet() throws IOException { + servlet.doGet(req, resp); + return new String(respOut.toByteArray(), "UTF-8"); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/IsRouterActiveServlet.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/IsRouterActiveServlet.java new file mode 100644 index 0000000000..cd84fced53 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/IsRouterActiveServlet.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import org.apache.hadoop.http.IsActiveServlet; + +import javax.servlet.ServletContext; + +/** + * Detect if the Router is active and ready to serve requests. + */ +public class IsRouterActiveServlet extends IsActiveServlet { + + @Override + protected boolean isActive() { + final ServletContext context = getServletContext(); + final Router router = RouterHttpServer.getRouterFromContext(context); + final RouterServiceState routerState = router.getRouterState(); + + return routerState == RouterServiceState.RUNNING; + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java index 21a9871727..0cf884f15c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java @@ -27,6 +27,8 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.service.AbstractService; +import javax.servlet.ServletContext; + /** * Web interface for the {@link Router}. It exposes the Web UI and the WebHDFS * methods from {@link RouterWebHdfsMethods}. @@ -116,6 +118,9 @@ public class RouterHttpServer extends AbstractService { private static void setupServlets( HttpServer2 httpServer, Configuration conf) { // TODO Add servlets for FSCK, etc + httpServer.addInternalServlet(IsRouterActiveServlet.SERVLET_NAME, + IsRouterActiveServlet.PATH_SPEC, + IsRouterActiveServlet.class); } public InetSocketAddress getHttpAddress() { @@ -125,4 +130,8 @@ public class RouterHttpServer extends AbstractService { public InetSocketAddress getHttpsAddress() { return this.httpsAddress; } + + public static Router getRouterFromContext(ServletContext context) { + return (Router)context.getAttribute(NAMENODE_ATTRIBUTE_KEY); + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md index c5bf5e1874..2f495875a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md @@ -62,7 +62,7 @@ Each Router has two roles: #### Federated interface The Router receives a client request, checks the State Store for the correct subcluster, and forwards the request to the active NameNode of that subcluster. The reply from the NameNode then flows in the opposite direction. -The Routers are stateless and can be behind a load balancer. +The Routers are stateless and can be behind a load balancer. For health checking, you can use /isActive endpoint as a health probe (e.g. http://ROUTER_HOSTNAME:ROUTER_PORT/isActive). For performance, the Router also caches remote mount table entries and the state of the subclusters. To make sure that changes have been propagated to all Routers, each Router heartbeats its state to the State Store. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/IsNameNodeActiveServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/IsNameNodeActiveServlet.java new file mode 100644 index 0000000000..1bd3f7f38f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/IsNameNodeActiveServlet.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import org.apache.hadoop.http.IsActiveServlet; + +/** + * Used by Load Balancers to find the active NameNode. + */ +public class IsNameNodeActiveServlet extends IsActiveServlet { + + @Override + protected boolean isActive() { + NameNode namenode = NameNodeHttpServer.getNameNodeFromContext( + getServletContext()); + return namenode.isActiveState(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index 861afae5c7..ae9c7feca7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -294,6 +294,9 @@ public class NameNodeHttpServer { true); httpServer.addInternalServlet("imagetransfer", ImageServlet.PATH_SPEC, ImageServlet.class, true); + httpServer.addInternalServlet(IsNameNodeActiveServlet.SERVLET_NAME, + IsNameNodeActiveServlet.PATH_SPEC, + IsNameNodeActiveServlet.class); } static FSImage getFsImageFromContext(ServletContext context) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md index f32868acc0..e4363fbec7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md @@ -423,6 +423,14 @@ This guide describes high-level uses of each of these subcommands. For specific **Note:** This is not yet implemented, and at present will always return success, unless the given NameNode is completely down. + +### Load Balancer Setup + +If you are running a set of NameNodes behind a Load Balancer (e.g. [Azure](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-custom-probe-overview) or [AWS](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-healthchecks.html) ) and would like the Load Balancer to point to the active NN, you can use the /isActive HTTP endpoint as a health probe. +http://NN_HOSTNAME/isActive will return a 200 status code response if the NN is in Active HA State, 405 otherwise. + + + Automatic Failover ------------------ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/IsResourceManagerActiveServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/IsResourceManagerActiveServlet.java new file mode 100644 index 0000000000..18e009d820 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/IsResourceManagerActiveServlet.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager; + +import org.apache.hadoop.http.IsActiveServlet; +import org.apache.hadoop.ha.HAServiceProtocol; + +/** + * Used by Load Balancers to find the active ResourceManager. + */ +public class IsResourceManagerActiveServlet extends IsActiveServlet { + + public static final String RM_ATTRIBUTE = "rm"; + + @Override + protected boolean isActive() { + ResourceManager rm = (ResourceManager) + getServletContext().getAttribute(RM_ATTRIBUTE); + RMContext rmContext = rm.getRMContext(); + HAServiceProtocol.HAServiceState state = rmContext.getHAServiceState(); + return state == HAServiceProtocol.HAServiceState.ACTIVE; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index bdda8717f6..d591f944d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -1206,6 +1206,11 @@ public class ResourceManager extends CompositeService } } + builder.withAttribute(IsResourceManagerActiveServlet.RM_ATTRIBUTE, this); + builder.withServlet(IsResourceManagerActiveServlet.SERVLET_NAME, + IsResourceManagerActiveServlet.PATH_SPEC, + IsResourceManagerActiveServlet.class); + webApp = builder.start(new RMWebApp(this), uiWebAppContext); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java index 1e4caba2ff..40b383478c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java @@ -37,6 +37,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HtmlQuoting; +import org.apache.hadoop.http.IsActiveServlet; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -68,7 +69,7 @@ public class RMWebAppFilter extends GuiceContainer { // define a set of URIs which do not need to do redirection private static final Set NON_REDIRECTED_URIS = Sets.newHashSet( - "/conf", "/stacks", "/logLevel", "/logs"); + "/conf", "/stacks", "/logLevel", "/logs", IsActiveServlet.PATH_SPEC); private String path; private boolean ahsEnabled; private String ahsPageURLPrefix; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md index ff9732887c..bde5ef2625 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md @@ -144,3 +144,8 @@ Assuming a standby RM is up and running, the Standby automatically redirects all ### Web Services Assuming a standby RM is up and running, RM web-services described at [ResourceManager REST APIs](./ResourceManagerRest.html) when invoked on a standby RM are automatically redirected to the Active RM. + +### Load Balancer Setup + +If you are running a set of ResourceManagers behind a Load Balancer (e.g. [Azure](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-custom-probe-overview) or [AWS](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-healthchecks.html) ) and would like the Load Balancer to point to the active RM, you can use the /isActive HTTP endpoint as a health probe. +http://RM_HOSTNAME/isActive will return a 200 status code response if the RM is in Active HA State, 405 otherwise.