HADOOP-15707. Add IsActiveServlet to be used for Load Balancers. Contributed by Lukas Majercak.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/IsActiveServlet.java

@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.http;
+
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+/**
+ * Used by Load Balancers to detect the active NameNode/ResourceManager/Router.
+ */
+public abstract class IsActiveServlet extends HttpServlet {
+
+  /** Default serial identifier. */
+  private static final long serialVersionUID = 1L;
+
+  public static final String SERVLET_NAME = "isActive";
+  public static final String PATH_SPEC = "/isActive";
+
+  public static final String RESPONSE_ACTIVE =
+      "I am Active!";
+
+  public static final String RESPONSE_NOT_ACTIVE =
+      "I am not Active!";
+
+  /**
+   * Check whether this instance is the Active one.
+   * @param req HTTP request
+   * @param resp HTTP response to write to
+   */
+  @Override
+  public void doGet(
+      final HttpServletRequest req, final HttpServletResponse resp)
+      throws IOException {
+
+    // By default requests are persistent. We don't want long-lived connections
+    // on server side.
+    resp.addHeader("Connection", "close");
+
+    if (!isActive()) {
+      // Report not SC_OK
+      resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED,
+          RESPONSE_NOT_ACTIVE);
+      return;
+    }
+    resp.setStatus(HttpServletResponse.SC_OK);
+    resp.getWriter().write(RESPONSE_ACTIVE);
+    resp.getWriter().flush();
+  }
+
+  /**
+   * @return true if this instance is in Active HA state.
+   */
+  protected abstract boolean isActive();
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestIsActiveServlet.java

@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.http;
+
+
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+
+/**
+ * Test if the {@link IsActiveServlet} returns the right answer if the
+ * underlying service is active.
+ */
+public class TestIsActiveServlet {
+
+  private IsActiveServlet servlet;
+  private HttpServletRequest req;
+  private HttpServletResponse resp;
+  private ByteArrayOutputStream respOut;
+
+  @Before
+  public void setUp() throws Exception {
+    req = mock(HttpServletRequest.class);
+    resp = mock(HttpServletResponse.class);
+    respOut = new ByteArrayOutputStream();
+    PrintWriter writer = new PrintWriter(respOut);
+    when(resp.getWriter()).thenReturn(writer);
+  }
+
+  @Test
+  public void testSucceedsOnActive() throws IOException {
+    servlet = new IsActiveServlet() {
+      @Override
+      protected boolean isActive() {
+        return true;
+      }
+    };
+
+    String response = doGet();
+    verify(resp, never()).sendError(anyInt(), anyString());
+    assertEquals(IsActiveServlet.RESPONSE_ACTIVE, response);
+  }
+
+  @Test
+  public void testFailsOnInactive() throws IOException {
+    servlet = new IsActiveServlet() {
+      @Override
+      protected boolean isActive() {
+        return false;
+      }
+    };
+
+    doGet();
+    verify(resp, atLeastOnce()).sendError(
+        eq(HttpServletResponse.SC_METHOD_NOT_ALLOWED),
+        eq(IsActiveServlet.RESPONSE_NOT_ACTIVE));
+  }
+
+  private String doGet() throws IOException {
+    servlet.doGet(req, resp);
+    return new String(respOut.toByteArray(), "UTF-8");
+  }
+}

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/IsRouterActiveServlet.java

@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.http.IsActiveServlet;
+
+import javax.servlet.ServletContext;
+
+/**
+ * Detect if the Router is active and ready to serve requests.
+ */
+public class IsRouterActiveServlet extends IsActiveServlet {
+
+  @Override
+  protected boolean isActive() {
+    final ServletContext context = getServletContext();
+    final Router router = RouterHttpServer.getRouterFromContext(context);
+    final RouterServiceState routerState = router.getRouterState();
+
+    return routerState == RouterServiceState.RUNNING;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java

@@ -27,6 +27,8 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.service.AbstractService;
 
+import javax.servlet.ServletContext;
+
 /**
  * Web interface for the {@link Router}. It exposes the Web UI and the WebHDFS
  * methods from {@link RouterWebHdfsMethods}.
@@ -116,6 +118,9 @@ public class RouterHttpServer extends AbstractService {
   private static void setupServlets(
       HttpServer2 httpServer, Configuration conf) {
     // TODO Add servlets for FSCK, etc
+    httpServer.addInternalServlet(IsRouterActiveServlet.SERVLET_NAME,
+        IsRouterActiveServlet.PATH_SPEC,
+        IsRouterActiveServlet.class);
   }
 
   public InetSocketAddress getHttpAddress() {
@@ -125,4 +130,8 @@ public class RouterHttpServer extends AbstractService {
   public InetSocketAddress getHttpsAddress() {
     return this.httpsAddress;
   }
+
+  public static Router getRouterFromContext(ServletContext context) {
+    return (Router)context.getAttribute(NAMENODE_ATTRIBUTE_KEY);
+  }
 }

hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md

@@ -62,7 +62,7 @@ Each Router has two roles:
 #### Federated interface
 The Router receives a client request, checks the State Store for the correct subcluster, and forwards the request to the active NameNode of that subcluster.
 The reply from the NameNode then flows in the opposite direction.
-The Routers are stateless and can be behind a load balancer.
+The Routers are stateless and can be behind a load balancer. For health checking, you can use /isActive endpoint as a health probe (e.g. http://ROUTER_HOSTNAME:ROUTER_PORT/isActive).
 For performance, the Router also caches remote mount table entries and the state of the subclusters.
 To make sure that changes have been propagated to all Routers, each Router heartbeats its state to the State Store.
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/IsNameNodeActiveServlet.java

@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.http.IsActiveServlet;
+
+/**
+ * Used by Load Balancers to find the active NameNode.
+ */
+public class IsNameNodeActiveServlet extends IsActiveServlet {
+
+  @Override
+  protected boolean isActive() {
+    NameNode namenode = NameNodeHttpServer.getNameNodeFromContext(
+        getServletContext());
+    return namenode.isActiveState();
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java

@@ -294,6 +294,9 @@ public class NameNodeHttpServer {
         true);
     httpServer.addInternalServlet("imagetransfer", ImageServlet.PATH_SPEC,
         ImageServlet.class, true);
+    httpServer.addInternalServlet(IsNameNodeActiveServlet.SERVLET_NAME,
+        IsNameNodeActiveServlet.PATH_SPEC,
+        IsNameNodeActiveServlet.class);
   }
 
   static FSImage getFsImageFromContext(ServletContext context) {

hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md

@@ -423,6 +423,14 @@ This guide describes high-level uses of each of these subcommands. For specific
     **Note:** This is not yet implemented, and at present will always return
     success, unless the given NameNode is completely down.
 
+
+### Load Balancer Setup
+
+If you are running a set of NameNodes behind a Load Balancer (e.g. [Azure](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-custom-probe-overview) or [AWS](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-healthchecks.html) ) and would like the Load Balancer to point to the active NN, you can use the /isActive HTTP endpoint as a health probe.
+http://NN_HOSTNAME/isActive will return a 200 status code response if the NN is in Active HA State, 405 otherwise.
+
+
+
 Automatic Failover
 ------------------
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/IsResourceManagerActiveServlet.java

@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import org.apache.hadoop.http.IsActiveServlet;
+import org.apache.hadoop.ha.HAServiceProtocol;
+
+/**
+ * Used by Load Balancers to find the active ResourceManager.
+ */
+public class IsResourceManagerActiveServlet extends IsActiveServlet {
+
+  public static final String RM_ATTRIBUTE = "rm";
+
+  @Override
+  protected boolean isActive() {
+    ResourceManager rm = (ResourceManager)
+        getServletContext().getAttribute(RM_ATTRIBUTE);
+    RMContext rmContext = rm.getRMContext();
+    HAServiceProtocol.HAServiceState state = rmContext.getHAServiceState();
+    return state == HAServiceProtocol.HAServiceState.ACTIVE;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java

@@ -1206,6 +1206,11 @@ public class ResourceManager extends CompositeService
       }
     }
 
+    builder.withAttribute(IsResourceManagerActiveServlet.RM_ATTRIBUTE, this);
+    builder.withServlet(IsResourceManagerActiveServlet.SERVLET_NAME,
+        IsResourceManagerActiveServlet.PATH_SPEC,
+        IsResourceManagerActiveServlet.class);
+
     webApp = builder.start(new RMWebApp(this), uiWebAppContext);
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java

@@ -37,6 +37,7 @@ import javax.servlet.http.HttpServletResponse;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HtmlQuoting;
+import org.apache.hadoop.http.IsActiveServlet;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -68,7 +69,7 @@ public class RMWebAppFilter extends GuiceContainer {
 
   // define a set of URIs which do not need to do redirection
   private static final Set<String> NON_REDIRECTED_URIS = Sets.newHashSet(
-      "/conf", "/stacks", "/logLevel", "/logs");
+      "/conf", "/stacks", "/logLevel", "/logs", IsActiveServlet.PATH_SPEC);
   private String path;
   private boolean ahsEnabled;
   private String ahsPageURLPrefix;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md

@@ -144,3 +144,8 @@ Assuming a standby RM is up and running, the Standby automatically redirects all
 ### Web Services
 
 Assuming a standby RM is up and running, RM web-services described at [ResourceManager REST APIs](./ResourceManagerRest.html) when invoked on a standby RM are automatically redirected to the Active RM.
+
+### Load Balancer Setup
+
+If you are running a set of ResourceManagers behind a Load Balancer (e.g. [Azure](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-custom-probe-overview) or [AWS](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-healthchecks.html) ) and would like the Load Balancer to point to the active RM, you can use the /isActive HTTP endpoint as a health probe.
+http://RM_HOSTNAME/isActive will return a 200 status code response if the RM is in Active HA State, 405 otherwise.