From e49162f4b3791dbf51079e3b19dd0c8bc2a85158 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Fri, 31 May 2019 10:28:09 +0530 Subject: [PATCH] YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by Zoltan Siegl. --- .../api/records/timeline/TimelineHealth.java | 82 +++++++++++++++++++ .../DocumentStoreTimelineReaderImpl.java | 13 +++ .../storage/HBaseTimelineReaderImpl.java | 13 +++ .../reader/TimelineReaderManager.java | 10 +++ .../reader/TimelineReaderWebServices.java | 33 ++++++++ .../storage/FileSystemTimelineReaderImpl.java | 15 ++++ .../storage/NoOpTimelineReaderImpl.java | 7 ++ .../storage/TimelineReader.java | 8 ++ .../reader/TestTimelineReaderWebServices.java | 19 +++++ 9 files changed, 200 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java new file mode 100644 index 0000000000..d592167b86 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records.timeline; + + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * This class holds health information for ATS. + */ +@XmlRootElement(name = "health") +@XmlAccessorType(XmlAccessType.NONE) +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class TimelineHealth { + + /** + * Timline health status. + * + * RUNNING - Service is up and running + * READER_CONNECTION_FAULURE - isConnectionAlive() of reader implementation + * reported an error + */ + public enum TimelineHealthStatus { + RUNNING, + READER_CONNECTION_FAILURE + } + + private TimelineHealthStatus healthStatus; + private String diagnosticsInfo; + + public TimelineHealth(TimelineHealthStatus healthy, String diagnosticsInfo) { + this.healthStatus = healthy; + this.diagnosticsInfo = diagnosticsInfo; + } + + public TimelineHealth() { + + } + + @XmlElement(name = "healthStatus") + public TimelineHealthStatus getHealthStatus() { + return healthStatus; + } + + @XmlElement(name = "diagnosticsInfo") + public String getDiagnosticsInfo() { + return diagnosticsInfo; + } + + + public void setHealthStatus(TimelineHealthStatus healthStatus) { + this.healthStatus = healthStatus; + } + + public void setDiagnosticsInfo(String diagnosticsInfo) { + this.diagnosticsInfo = diagnosticsInfo; + } + + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineReaderImpl.java index 2159132585..8de3b8645a 100755 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineReaderImpl.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.server.timelineservice.documentstore.lib.DocumentStoreVendor; @@ -100,6 +101,18 @@ public Set getEntityTypes(TimelineReaderContext context) { return collectionReader.fetchEntityTypes(context); } + @Override + public TimelineHealth getHealthStatus() { + if (collectionReader != null) { + return new TimelineHealth(TimelineHealth.TimelineHealthStatus.RUNNING, + ""); + } else { + return new TimelineHealth( + TimelineHealth.TimelineHealthStatus.READER_CONNECTION_FAILURE, + "Timeline store reader not initialized."); + } + } + // for honoring all filters from {@link TimelineEntityFilters} private Set applyFilters(TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java index d00ae4b5f7..653126e100 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -158,6 +159,18 @@ public Set getEntityTypes(TimelineReaderContext context) return reader.readEntityTypes(hbaseConf, conn); } + @Override + public TimelineHealth getHealthStatus() { + if (!this.isHBaseDown()) { + return new TimelineHealth(TimelineHealth.TimelineHealthStatus.RUNNING, + ""); + } else { + return new TimelineHealth( + TimelineHealth.TimelineHealthStatus.READER_CONNECTION_FAILURE, + "HBase connection is down"); + } + } + protected static final TimelineEntityFilters MONITOR_FILTERS = new TimelineEntityFilters.Builder().entityLimit(1L).build(); protected static final TimelineDataToRetrieve DATA_TO_RETRIEVE = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java index 8c7c974b5b..06da543fdb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth; import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity; import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; @@ -219,4 +220,13 @@ public boolean checkAccess(UserGroupInformation callerUGI) { } return callerUGI != null && adminACLsManager.isAdmin(callerUGI); } + + /** + * Check if reader connection is alive. + * + * @return boolean True if reader connection is alive, false otherwise. + */ + public TimelineHealth getHealthStatus() { + return reader.getHealthStatus(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java index 330e1f4873..7433692336 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java @@ -48,6 +48,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; +import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth; import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; @@ -218,6 +219,38 @@ public TimelineAbout about( return TimelineUtils.createTimelineAbout("Timeline Reader API"); } + /** + * Health check REST end point. + * + * @param req Servlet request. + * @param res Servlet response. + * + * @return A {@link Response} object with HTTP status 200 OK if the service + * is running. + * Otherwise, a {@link Response} object with HTTP status 500 is + * returned. + */ + @GET + @Path("/health") + @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) + public Response health( + @Context HttpServletRequest req, + @Context HttpServletResponse res + ) { + Response response; + TimelineHealth timelineHealth = this.getTimelineReaderManager().getHealthStatus(); + if (timelineHealth.getHealthStatus() + .equals(TimelineHealth.TimelineHealthStatus.RUNNING)) { + response = Response.ok(timelineHealth).build(); + } else { + LOG.info("Timeline services health check: timeline reader reported " + + "connection failure"); + response = Response.serverError().entity(timelineHealth).build(); + } + + return response; + } + /** * Return a single entity for a given entity type and UID which is a delimited * string containing clusterid, userid, flow name, flowrun id and app id. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java index ef08a9d9ef..012c9a18f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java @@ -46,6 +46,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; @@ -442,4 +443,18 @@ public Set getEntities(TimelineReaderContext context, } return result; } + + @Override + public TimelineHealth getHealthStatus() { + try { + fs.exists(rootPath); + } catch (IOException e) { + return new TimelineHealth( + TimelineHealth.TimelineHealthStatus.READER_CONNECTION_FAILURE, + e.getMessage() + ); + } + return new TimelineHealth(TimelineHealth.TimelineHealthStatus.RUNNING, + ""); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/NoOpTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/NoOpTimelineReaderImpl.java index bfa530984f..b2fd773153 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/NoOpTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/NoOpTimelineReaderImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; @@ -71,4 +72,10 @@ public Set getEntityTypes(TimelineReaderContext context) "requests would be empty"); return new HashSet<>(); } + + @Override + public TimelineHealth getHealthStatus() { + return new TimelineHealth(TimelineHealth.TimelineHealthStatus.RUNNING, + "NoOpTimelineReader is configured. "); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java index 16d623ab46..4bcc6dc5a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java @@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; @@ -192,4 +193,11 @@ Set getEntities( * storage. */ Set getEntityTypes(TimelineReaderContext context) throws IOException; + + /** + * Check if reader connection is working properly. + * + * @return True if reader connection works as expected, false otherwise. + */ + TimelineHealth getHealthStatus(); } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java index 03939ada8a..ef74716b56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java @@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; +import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -777,4 +778,22 @@ public void testGetContainer() throws Exception { client.destroy(); } } + + @Test + public void testHealthCheck() throws Exception { + Client client = createClient(); + try { + URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + "timeline/health"); + ClientResponse resp = getResponse(client, uri); + TimelineHealth timelineHealth = + resp.getEntity(new GenericType() { + }); + assertEquals(200, resp.getStatus()); + assertEquals(TimelineHealth.TimelineHealthStatus.RUNNING, + timelineHealth.getHealthStatus()); + } finally { + client.destroy(); + } + } } \ No newline at end of file