diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 359b4ed2ee..76d5ecce89 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -96,6 +96,9 @@ Release 2.5.0 - UNRELEASED YARN-1936. Added security support for the Timeline Client. (Zhijie Shen via vinodkv) + YARN-1937. Added owner-only ACLs support for Timeline Client and server. + (Zhijie Shen via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java index 37c00466ca..77a97ba95f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java @@ -107,6 +107,17 @@ public static class TimelinePutError { */ public static final int IO_EXCEPTION = 2; + /** + * Error code returned if the user specifies the timeline system reserved + * filter key + */ + public static final int SYSTEM_FILTER_CONFLICT = 3; + + /** + * Error code returned if the user is denied to access the timeline data + */ + public static final int ACCESS_DENIED = 4; + private String entityId; private String entityType; private int errorCode; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index a2f5a2489e..5e1277f75f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineAuthenticationFilterInitializer; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; @@ -63,6 +64,7 @@ public class ApplicationHistoryServer extends CompositeService { protected ApplicationHistoryManager historyManager; protected TimelineStore timelineStore; protected TimelineDelegationTokenSecretManagerService secretManagerService; + protected TimelineACLsManager timelineACLsManager; protected WebApp webApp; public ApplicationHistoryServer() { @@ -79,6 +81,7 @@ protected void serviceInit(Configuration conf) throws Exception { addIfService(timelineStore); secretManagerService = createTimelineDelegationTokenSecretManagerService(conf); addService(secretManagerService); + timelineACLsManager = createTimelineACLsManager(conf); DefaultMetricsSystem.initialize("ApplicationHistoryServer"); JvmMetrics.initSingleton("ApplicationHistoryServer", null); @@ -169,6 +172,10 @@ protected TimelineStore createTimelineStore( return new TimelineDelegationTokenSecretManagerService(); } + protected TimelineACLsManager createTimelineACLsManager(Configuration conf) { + return new TimelineACLsManager(conf); + } + protected void startWebApp() { Configuration conf = getConfig(); // Play trick to make the customized filter will only be loaded by the @@ -196,6 +203,7 @@ protected void startWebApp() { ahsWebApp.setApplicationHistoryManager(historyManager); ahsWebApp.setTimelineStore(timelineStore); ahsWebApp.setTimelineDelegationTokenSecretManagerService(secretManagerService); + ahsWebApp.setTimelineACLsManager(timelineACLsManager); webApp = WebApps .$for("applicationhistory", ApplicationHistoryClientService.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java index 86ac1f8110..06f3d607a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -40,8 +39,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; /** @@ -314,15 +313,29 @@ private static TimelineEntity maskFields( entityToReturn.setEntityId(entity.getEntityId()); entityToReturn.setEntityType(entity.getEntityType()); entityToReturn.setStartTime(entity.getStartTime()); - entityToReturn.setEvents(fields.contains(Field.EVENTS) ? - entity.getEvents() : fields.contains(Field.LAST_EVENT_ONLY) ? - Arrays.asList(entity.getEvents().get(0)) : null); - entityToReturn.setRelatedEntities(fields.contains(Field.RELATED_ENTITIES) ? - entity.getRelatedEntities() : null); - entityToReturn.setPrimaryFilters(fields.contains(Field.PRIMARY_FILTERS) ? - entity.getPrimaryFilters() : null); - entityToReturn.setOtherInfo(fields.contains(Field.OTHER_INFO) ? - entity.getOtherInfo() : null); + // Deep copy + if (fields.contains(Field.EVENTS)) { + entityToReturn.addEvents(entity.getEvents()); + } else if (fields.contains(Field.LAST_EVENT_ONLY)) { + entityToReturn.addEvent(entity.getEvents().get(0)); + } else { + entityToReturn.setEvents(null); + } + if (fields.contains(Field.RELATED_ENTITIES)) { + entityToReturn.addRelatedEntities(entity.getRelatedEntities()); + } else { + entityToReturn.setRelatedEntities(null); + } + if (fields.contains(Field.PRIMARY_FILTERS)) { + entityToReturn.addPrimaryFilters(entity.getPrimaryFilters()); + } else { + entityToReturn.setPrimaryFilters(null); + } + if (fields.contains(Field.OTHER_INFO)) { + entityToReturn.addOtherInfo(entity.getOtherInfo()); + } else { + entityToReturn.setOtherInfo(null); + } return entityToReturn; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java index 6b50d832cf..fc02873beb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java @@ -18,12 +18,25 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; -@InterfaceAudience.Private -@InterfaceStability.Unstable +@Private +@Unstable public interface TimelineStore extends Service, TimelineReader, TimelineWriter { + + /** + * The system filter which will be automatically added to a + * {@link TimelineEntity}'s primary filter section when storing the entity. + * The filter key is case sensitive. Users are supposed not to use the key + * reserved by the timeline system. + */ + @Private + enum SystemFilter { + ENTITY_OWNER + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java new file mode 100644 index 0000000000..5bc8705222 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import java.io.IOException; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore.SystemFilter; + +import com.google.common.annotations.VisibleForTesting; + +/** + * TimelineACLsManager check the entity level timeline data access. + */ +@Private +public class TimelineACLsManager { + + private static final Log LOG = LogFactory.getLog(TimelineACLsManager.class); + + private boolean aclsEnabled; + + public TimelineACLsManager(Configuration conf) { + aclsEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, + YarnConfiguration.DEFAULT_YARN_ACL_ENABLE); + } + + public boolean checkAccess(UserGroupInformation callerUGI, + TimelineEntity entity) throws YarnException, IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Verifying the access of " + callerUGI.getShortUserName() + + " on the timeline entity " + + new EntityIdentifier(entity.getEntityId(), entity.getEntityType())); + } + + if (!aclsEnabled) { + return true; + } + + Set values = + entity.getPrimaryFilters().get( + SystemFilter.ENTITY_OWNER.toString()); + if (values == null || values.size() != 1) { + throw new YarnException("Owner information of the timeline entity " + + new EntityIdentifier(entity.getEntityId(), entity.getEntityType()) + + " is corrupted."); + } + String owner = values.iterator().next().toString(); + // TODO: Currently we just check the user is the timeline entity owner. In + // the future, we need to check whether the user is admin or is in the + // allowed user/group list + if (callerUGI != null && callerUGI.getShortUserName().equals(owner)) { + return true; + } + return false; + } + + @Private + @VisibleForTesting + public void setACLsEnabled(boolean aclsEnabled) { + this.aclsEnabled = aclsEnabled; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java index c3a19d476b..17b1e6286d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; @@ -36,6 +37,7 @@ public class AHSWebApp extends WebApp implements YarnWebParams { private ApplicationHistoryManager applicationHistoryManager; private TimelineStore timelineStore; private TimelineDelegationTokenSecretManagerService secretManagerService; + private TimelineACLsManager timelineACLsManager; private static AHSWebApp instance = null; @@ -83,6 +85,14 @@ public void setTimelineDelegationTokenSecretManagerService( this.secretManagerService = secretManagerService; } + public TimelineACLsManager getTimelineACLsManager() { + return timelineACLsManager; + } + + public void setTimelineACLsManager(TimelineACLsManager timelineACLsManager) { + this.timelineACLsManager = timelineACLsManager; + } + @Override public void setup() { bind(YarnJacksonJaxbJsonProvider.class); @@ -93,6 +103,7 @@ public void setup() { bind(TimelineStore.class).toInstance(timelineStore); bind(TimelineDelegationTokenSecretManagerService.class).toInstance( secretManagerService); + bind(TimelineACLsManager.class).toInstance(timelineACLsManager); route("/", AHSController.class); route(pajoin("/apps", APP_STATE), AHSController.class); route(pajoin("/app", APPLICATION_ID), AHSController.class, "app"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java index 44567ea229..6041779b13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java @@ -25,6 +25,7 @@ import java.util.Collection; import java.util.EnumSet; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.SortedSet; @@ -52,17 +53,21 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.NameValuePair; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; +import org.apache.hadoop.yarn.webapp.NotFoundException; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -75,10 +80,13 @@ public class TimelineWebServices { private static final Log LOG = LogFactory.getLog(TimelineWebServices.class); private TimelineStore store; + private TimelineACLsManager timelineACLsManager; @Inject - public TimelineWebServices(TimelineStore store) { + public TimelineWebServices(TimelineStore store, + TimelineACLsManager timelineACLsManager) { this.store = store; + this.timelineACLsManager = timelineACLsManager; } @XmlRootElement(name = "about") @@ -141,6 +149,9 @@ public TimelineEntities getEntities( init(res); TimelineEntities entities = null; try { + EnumSet fieldEnums = parseFieldsStr(fields, ","); + boolean modified = extendFields(fieldEnums); + UserGroupInformation callerUGI = getUser(req); entities = store.getEntities( parseStr(entityType), parseLongStr(limit), @@ -150,7 +161,33 @@ public TimelineEntities getEntities( parseLongStr(fromTs), parsePairStr(primaryFilter, ":"), parsePairsStr(secondaryFilter, ",", ":"), - parseFieldsStr(fields, ",")); + fieldEnums); + if (entities != null) { + Iterator entitiesItr = + entities.getEntities().iterator(); + while (entitiesItr.hasNext()) { + TimelineEntity entity = entitiesItr.next(); + try { + // check ACLs + if (!timelineACLsManager.checkAccess(callerUGI, entity)) { + entitiesItr.remove(); + } else { + // clean up system data + if (modified) { + entity.setPrimaryFilters(null); + } else { + cleanupOwnerInfo(entity); + } + } + } catch (YarnException e) { + LOG.error("Error when verifying access for user " + callerUGI + + " on the events of the timeline entity " + + new EntityIdentifier(entity.getEntityId(), + entity.getEntityType()), e); + entitiesItr.remove(); + } + } + } } catch (NumberFormatException e) { throw new BadRequestException( "windowStart, windowEnd or limit is not a numeric value."); @@ -182,9 +219,25 @@ public TimelineEntity getEntity( init(res); TimelineEntity entity = null; try { + EnumSet fieldEnums = parseFieldsStr(fields, ","); + boolean modified = extendFields(fieldEnums); entity = store.getEntity(parseStr(entityId), parseStr(entityType), - parseFieldsStr(fields, ",")); + fieldEnums); + if (entity != null) { + // check ACLs + UserGroupInformation callerUGI = getUser(req); + if (!timelineACLsManager.checkAccess(callerUGI, entity)) { + entity = null; + } else { + // clean up the system data + if (modified) { + entity.setPrimaryFilters(null); + } else { + cleanupOwnerInfo(entity); + } + } + } } catch (IllegalArgumentException e) { throw new BadRequestException( "requested invalid field."); @@ -192,9 +245,15 @@ public TimelineEntity getEntity( LOG.error("Error getting entity", e); throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); + } catch (YarnException e) { + LOG.error("Error getting entity", e); + throw new WebApplicationException(e, + Response.Status.INTERNAL_SERVER_ERROR); } if (entity == null) { - throw new WebApplicationException(Response.Status.NOT_FOUND); + throw new NotFoundException("Timeline entity " + + new EntityIdentifier(parseStr(entityId), parseStr(entityType)) + + " is not found"); } return entity; } @@ -217,6 +276,7 @@ public TimelineEvents getEvents( init(res); TimelineEvents events = null; try { + UserGroupInformation callerUGI = getUser(req); events = store.getEntityTimelines( parseStr(entityType), parseArrayStr(entityId, ","), @@ -224,6 +284,29 @@ public TimelineEvents getEvents( parseLongStr(windowStart), parseLongStr(windowEnd), parseArrayStr(eventType, ",")); + if (events != null) { + Iterator eventsItr = + events.getAllEvents().iterator(); + while (eventsItr.hasNext()) { + TimelineEvents.EventsOfOneEntity eventsOfOneEntity = eventsItr.next(); + try { + TimelineEntity entity = store.getEntity( + eventsOfOneEntity.getEntityId(), + eventsOfOneEntity.getEntityType(), + EnumSet.of(Field.PRIMARY_FILTERS)); + // check ACLs + if (!timelineACLsManager.checkAccess(callerUGI, entity)) { + eventsItr.remove(); + } + } catch (Exception e) { + LOG.error("Error when verifying access for user " + callerUGI + + " on the events of the timeline entity " + + new EntityIdentifier(eventsOfOneEntity.getEntityId(), + eventsOfOneEntity.getEntityType()), e); + eventsItr.remove(); + } + } + } } catch (NumberFormatException e) { throw new BadRequestException( "windowStart, windowEnd or limit is not a numeric value."); @@ -252,12 +335,61 @@ public TimelinePutResponse postEntities( if (entities == null) { return new TimelinePutResponse(); } + UserGroupInformation callerUGI = getUser(req); try { List entityIDs = new ArrayList(); + TimelineEntities entitiesToPut = new TimelineEntities(); + List errors = + new ArrayList(); for (TimelineEntity entity : entities.getEntities()) { EntityIdentifier entityID = new EntityIdentifier(entity.getEntityId(), entity.getEntityType()); + + // check if there is existing entity + try { + TimelineEntity existingEntity = + store.getEntity(entityID.getId(), entityID.getType(), + EnumSet.of(Field.PRIMARY_FILTERS)); + if (existingEntity != null + && !timelineACLsManager.checkAccess(callerUGI, existingEntity)) { + throw new YarnException("The timeline entity " + entityID + + " was not put by " + callerUGI + " before"); + } + } catch (Exception e) { + // Skip the entity which already exists and was put by others + LOG.warn("Skip the timeline entity: " + entityID + ", because " + + e.getMessage()); + TimelinePutResponse.TimelinePutError error = + new TimelinePutResponse.TimelinePutError(); + error.setEntityId(entityID.getId()); + error.setEntityType(entityID.getType()); + error.setErrorCode( + TimelinePutResponse.TimelinePutError.ACCESS_DENIED); + errors.add(error); + continue; + } + + // inject owner information for the access check + try { + injectOwnerInfo(entity, + callerUGI == null ? "" : callerUGI.getShortUserName()); + } catch (YarnException e) { + // Skip the entity which messes up the primary filter and record the + // error + LOG.warn("Skip the timeline entity: " + entityID + ", because " + + e.getMessage()); + TimelinePutResponse.TimelinePutError error = + new TimelinePutResponse.TimelinePutError(); + error.setEntityId(entityID.getId()); + error.setEntityType(entityID.getType()); + error.setErrorCode( + TimelinePutResponse.TimelinePutError.SYSTEM_FILTER_CONFLICT); + errors.add(error); + continue; + } + entityIDs.add(entityID); + entitiesToPut.addEntity(entity); if (LOG.isDebugEnabled()) { LOG.debug("Storing the entity " + entityID + ", JSON-style content: " + TimelineUtils.dumpTimelineRecordtoJSON(entity)); @@ -266,7 +398,10 @@ public TimelinePutResponse postEntities( if (LOG.isDebugEnabled()) { LOG.debug("Storing entities: " + CSV_JOINER.join(entityIDs)); } - return store.put(entities); + TimelinePutResponse response = store.put(entitiesToPut); + // add the errors of timeline system filter key conflict + response.addErrors(errors); + return response; } catch (IOException e) { LOG.error("Error putting entities", e); throw new WebApplicationException(e, @@ -350,6 +485,14 @@ private static EnumSet parseFieldsStr(String str, String delimiter) { } } + private static boolean extendFields(EnumSet fieldEnums) { + boolean modified = false; + if (fieldEnums != null && !fieldEnums.contains(Field.PRIMARY_FILTERS)) { + fieldEnums.add(Field.PRIMARY_FILTERS); + modified = true; + } + return modified; + } private static Long parseLongStr(String str) { return str == null ? null : Long.parseLong(str.trim()); } @@ -358,4 +501,34 @@ private static String parseStr(String str) { return str == null ? null : str.trim(); } + private static UserGroupInformation getUser(HttpServletRequest req) { + String remoteUser = req.getRemoteUser(); + UserGroupInformation callerUGI = null; + if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); + } + return callerUGI; + } + + private static void injectOwnerInfo(TimelineEntity timelineEntity, + String owner) throws YarnException { + if (timelineEntity.getPrimaryFilters() != null && + timelineEntity.getPrimaryFilters().containsKey( + TimelineStore.SystemFilter.ENTITY_OWNER)) { + throw new YarnException( + "User should not use the timeline system filter key: " + + TimelineStore.SystemFilter.ENTITY_OWNER); + } + timelineEntity.addPrimaryFilter( + TimelineStore.SystemFilter.ENTITY_OWNER + .toString(), owner); + } + + private static void cleanupOwnerInfo(TimelineEntity timelineEntity) { + if (timelineEntity.getPrimaryFilters() != null) { + timelineEntity.getPrimaryFilters().remove( + TimelineStore.SystemFilter.ENTITY_OWNER.toString()); + } + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java new file mode 100644 index 0000000000..2c536681ae --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.junit.Assert; +import org.junit.Test; + +public class TestTimelineACLsManager { + + @Test + public void testYarnACLsNotEnabled() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); + TimelineACLsManager timelineACLsManager = + new TimelineACLsManager(conf); + TimelineEntity entity = new TimelineEntity(); + entity.addPrimaryFilter( + TimelineStore.SystemFilter.ENTITY_OWNER + .toString(), "owner"); + Assert.assertTrue( + "Always true when ACLs are not enabled", + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("user"), entity)); + } + + @Test + public void testYarnACLsEnabled() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + TimelineACLsManager timelineACLsManager = + new TimelineACLsManager(conf); + TimelineEntity entity = new TimelineEntity(); + entity.addPrimaryFilter( + TimelineStore.SystemFilter.ENTITY_OWNER + .toString(), "owner"); + Assert.assertTrue( + "Owner should be allowed to access", + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("owner"), entity)); + Assert.assertFalse( + "Other shouldn't be allowed to access", + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("other"), entity)); + } + + @Test + public void testCorruptedOwnerInfo() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + TimelineACLsManager timelineACLsManager = + new TimelineACLsManager(conf); + TimelineEntity entity = new TimelineEntity(); + try { + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("owner"), entity); + Assert.fail("Exception is expected"); + } catch (YarnException e) { + Assert.assertTrue("It's not the exact expected exception", e.getMessage() + .contains("is corrupted.")); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java index 650184bd51..9b0ae3761d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java @@ -20,19 +20,32 @@ import static org.junit.Assert.assertEquals; +import java.io.IOException; + +import javax.inject.Singleton; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; import javax.ws.rs.core.MediaType; -import org.junit.Assert; - +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TestMemoryTimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.junit.Assert; import org.junit.Test; import com.google.inject.Guice; @@ -50,6 +63,8 @@ public class TestTimelineWebServices extends JerseyTest { private static TimelineStore store; + private static TimelineACLsManager timelineACLsManager; + private static String remoteUser; private long beforeTime; private Injector injector = Guice.createInjector(new ServletModule() { @@ -65,7 +80,12 @@ protected void configureServlets() { Assert.fail(); } bind(TimelineStore.class).toInstance(store); + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); + timelineACLsManager = new TimelineACLsManager(conf); + bind(TimelineACLsManager.class).toInstance(timelineACLsManager); serve("/*").with(GuiceContainer.class); + filter("/*").through(TestFilter.class); } }); @@ -340,8 +360,8 @@ public void testGetEvents() throws Exception { public void testPostEntities() throws Exception { TimelineEntities entities = new TimelineEntities(); TimelineEntity entity = new TimelineEntity(); - entity.setEntityId("test id"); - entity.setEntityType("test type"); + entity.setEntityId("test id 1"); + entity.setEntityType("test type 1"); entity.setStartTime(System.currentTimeMillis()); entities.addEntity(entity); WebResource r = resource(); @@ -355,14 +375,248 @@ public void testPostEntities() throws Exception { Assert.assertEquals(0, putResposne.getErrors().size()); // verify the entity exists in the store response = r.path("ws").path("v1").path("timeline") - .path("test type").path("test id") + .path("test type 1").path("test id 1") .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); entity = response.getEntity(TimelineEntity.class); Assert.assertNotNull(entity); - Assert.assertEquals("test id", entity.getEntityId()); - Assert.assertEquals("test type", entity.getEntityType()); + Assert.assertEquals("test id 1", entity.getEntityId()); + Assert.assertEquals("test type 1", entity.getEntityType()); } + @Test + public void testPostEntitiesWithYarnACLsEnabled() throws Exception { + timelineACLsManager.setACLsEnabled(true); + remoteUser = "tester"; + try { + TimelineEntities entities = new TimelineEntities(); + TimelineEntity entity = new TimelineEntity(); + entity.setEntityId("test id 2"); + entity.setEntityType("test type 2"); + entity.setStartTime(System.currentTimeMillis()); + entities.addEntity(entity); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class); + Assert.assertNotNull(putResponse); + Assert.assertEquals(0, putResponse.getErrors().size()); + + // override/append timeline data in the same entity with different user + remoteUser = "other"; + response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + putResponse = response.getEntity(TimelinePutResponse.class); + Assert.assertNotNull(putResponse); + Assert.assertEquals(1, putResponse.getErrors().size()); + Assert.assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED, + putResponse.getErrors().get(0).getErrorCode()); + } finally { + timelineACLsManager.setACLsEnabled(false); + remoteUser = null; + } + } + + @Test + public void testGetEntityWithYarnACLsEnabled() throws Exception { + timelineACLsManager.setACLsEnabled(true); + remoteUser = "tester"; + try { + TimelineEntities entities = new TimelineEntities(); + TimelineEntity entity = new TimelineEntity(); + entity.setEntityId("test id 3"); + entity.setEntityType("test type 3"); + entity.setStartTime(System.currentTimeMillis()); + entities.addEntity(entity); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + // verify the system data will not be exposed + // 1. No field specification + response = r.path("ws").path("v1").path("timeline") + .path("test type 3").path("test id 3") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + entity = response.getEntity(TimelineEntity.class); + Assert.assertNull(entity.getPrimaryFilters().get( + TimelineStore.SystemFilter.ENTITY_OWNER.toString())); + // 2. other field + response = r.path("ws").path("v1").path("timeline") + .path("test type 3").path("test id 3") + .queryParam("fields", "relatedentities") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + entity = response.getEntity(TimelineEntity.class); + Assert.assertNull(entity.getPrimaryFilters().get( + TimelineStore.SystemFilter.ENTITY_OWNER.toString())); + // 3. primaryfilters field + response = r.path("ws").path("v1").path("timeline") + .path("test type 3").path("test id 3") + .queryParam("fields", "primaryfilters") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + entity = response.getEntity(TimelineEntity.class); + Assert.assertNull(entity.getPrimaryFilters().get( + TimelineStore.SystemFilter.ENTITY_OWNER.toString())); + + // get entity with other user + remoteUser = "other"; + response = r.path("ws").path("v1").path("timeline") + .path("test type 3").path("test id 3") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + assertEquals(ClientResponse.Status.NOT_FOUND, + response.getClientResponseStatus()); + } finally { + timelineACLsManager.setACLsEnabled(false); + remoteUser = null; + } + } + + @Test + public void testGetEntitiesWithYarnACLsEnabled() { + timelineACLsManager.setACLsEnabled(true); + remoteUser = "tester"; + try { + TimelineEntities entities = new TimelineEntities(); + TimelineEntity entity = new TimelineEntity(); + entity.setEntityId("test id 4"); + entity.setEntityType("test type 4"); + entity.setStartTime(System.currentTimeMillis()); + entities.addEntity(entity); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + + remoteUser = "other"; + entities = new TimelineEntities(); + entity = new TimelineEntity(); + entity.setEntityId("test id 5"); + entity.setEntityType("test type 4"); + entity.setStartTime(System.currentTimeMillis()); + entities.addEntity(entity); + r = resource(); + response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + + response = r.path("ws").path("v1").path("timeline") + .path("test type 4") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + entities = response.getEntity(TimelineEntities.class); + assertEquals(1, entities.getEntities().size()); + assertEquals("test type 4", entities.getEntities().get(0).getEntityType()); + assertEquals("test id 5", entities.getEntities().get(0).getEntityId()); + } finally { + timelineACLsManager.setACLsEnabled(false); + remoteUser = null; + } + } + + @Test + public void testGetEventsWithYarnACLsEnabled() { + timelineACLsManager.setACLsEnabled(true); + remoteUser = "tester"; + try { + TimelineEntities entities = new TimelineEntities(); + TimelineEntity entity = new TimelineEntity(); + entity.setEntityId("test id 5"); + entity.setEntityType("test type 5"); + entity.setStartTime(System.currentTimeMillis()); + TimelineEvent event = new TimelineEvent(); + event.setEventType("event type 1"); + event.setTimestamp(System.currentTimeMillis()); + entity.addEvent(event); + entities.addEntity(entity); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + + remoteUser = "other"; + entities = new TimelineEntities(); + entity = new TimelineEntity(); + entity.setEntityId("test id 6"); + entity.setEntityType("test type 5"); + entity.setStartTime(System.currentTimeMillis()); + event = new TimelineEvent(); + event.setEventType("event type 2"); + event.setTimestamp(System.currentTimeMillis()); + entity.addEvent(event); + entities.addEntity(entity); + r = resource(); + response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + + response = r.path("ws").path("v1").path("timeline") + .path("test type 5").path("events") + .queryParam("entityId", "test id 5,test id 6") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + TimelineEvents events = response.getEntity(TimelineEvents.class); + assertEquals(1, events.getAllEvents().size()); + assertEquals("test id 6", events.getAllEvents().get(0).getEntityId()); + } finally { + timelineACLsManager.setACLsEnabled(false); + remoteUser = null; + } + } + + @Singleton + private static class TestFilter implements Filter { + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain chain) throws IOException, ServletException { + if (request instanceof HttpServletRequest) { + request = + new TestHttpServletRequestWrapper((HttpServletRequest) request); + } + chain.doFilter(request, response); + } + + @Override + public void destroy() { + } + + } + + private static class TestHttpServletRequestWrapper extends HttpServletRequestWrapper { + + public TestHttpServletRequestWrapper(HttpServletRequest request) { + super(request); + } + + @Override + public String getRemoteUser() { + return TestTimelineWebServices.remoteUser; + } + + } }