YARN-8455. Add basic ACL check for all ATS v2 REST APIs. Contributed by Rohith Sharma K S.

This commit is contained in:
Sunil G 2018-06-29 10:02:53 -07:00
parent 73746c5da7
commit 469b29c081
3 changed files with 407 additions and 38 deletions

View File

@ -0,0 +1,93 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.timelineservice.reader;
import java.util.List;
/**
* Used for decoding FROM_ID
*/
enum TimelineFromIdConverter {
APPLICATION_FROMID {
@Override TimelineReaderContext decodeUID(String fromId) throws Exception {
if (fromId == null) {
return null;
}
List<String> appTupleList = TimelineReaderUtils.split(fromId);
if (appTupleList == null || appTupleList.size() != 5) {
throw new IllegalArgumentException(
"Invalid row key for application table.");
}
return new TimelineReaderContext(appTupleList.get(0), appTupleList.get(1),
appTupleList.get(2), Long.parseLong(appTupleList.get(3)),
appTupleList.get(4), null, null);
}
},
SUB_APPLICATION_ENTITY_FROMID {
@Override TimelineReaderContext decodeUID(String fromId) throws Exception {
if (fromId == null) {
return null;
}
List<String> split = TimelineReaderUtils.split(fromId);
if (split == null || split.size() != 6) {
throw new IllegalArgumentException(
"Invalid row key for sub app table.");
}
String subAppUserId = split.get(0);
String clusterId = split.get(1);
String entityType = split.get(2);
Long entityIdPrefix = Long.valueOf(split.get(3));
String entityId = split.get(4);
String userId = split.get(5);
return new TimelineReaderContext(clusterId, userId, null, null, null,
entityType, entityIdPrefix, entityId, subAppUserId);
}
},
GENERIC_ENTITY_FROMID {
@Override TimelineReaderContext decodeUID(String fromId) throws Exception {
if (fromId == null) {
return null;
}
List<String> split = TimelineReaderUtils.split(fromId);
if (split == null || split.size() != 8) {
throw new IllegalArgumentException("Invalid row key for entity table.");
}
Long flowRunId = Long.valueOf(split.get(3));
Long entityIdPrefix = Long.valueOf(split.get(6));
return new TimelineReaderContext(split.get(0), split.get(1), split.get(2),
flowRunId, split.get(4), split.get(5), entityIdPrefix, split.get(7));
}
};
/**
* Decodes FROM_ID depending on FROM_ID implementation.
*
* @param fromId FROM_ID to be decoded.
* @return a {@link TimelineReaderContext} object if FROM_ID passed can be
* decoded, null otherwise.
* @throws Exception if any problem occurs while decoding.
*/
abstract TimelineReaderContext decodeUID(String fromId) throws Exception;
}

View File

@ -55,6 +55,7 @@
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import com.google.common.annotations.VisibleForTesting;
@ -188,6 +189,8 @@ private static void handleException(Exception e, String url, long startTime,
"Filter Parsing failed." : e.getMessage());
} else if (e instanceof BadRequestException) {
throw (BadRequestException)e;
} else if (e instanceof ForbiddenException) {
throw (ForbiddenException) e;
} else {
LOG.error("Error while processing REST request", e);
throw new WebApplicationException(e,
@ -339,6 +342,7 @@ public Set<TimelineEntity> getEntities(
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForGenericEntities(entities, callerUGI, entityType);
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
@ -607,13 +611,15 @@ public Set<TimelineEntity> getEntities(
.createTimelineReaderContext(clusterId, userId, flowName, flowRunId,
appId, entityType, null, null);
entities = timelineReaderManager.getEntities(context,
TimelineReaderWebServicesUtils.createTimelineEntityFilters(
limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
infofilters, conffilters, metricfilters, eventfilters,
fromId),
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
TimelineReaderWebServicesUtils
.createTimelineEntityFilters(limit, createdTimeStart,
createdTimeEnd, relatesTo, isRelatedTo, infofilters,
conffilters, metricfilters, eventfilters, fromId),
TimelineReaderWebServicesUtils
.createTimelineDataToRetrieve(confsToRetrieve, metricsToRetrieve,
fields, metricsLimit, metricsTimeStart, metricsTimeEnd));
checkAccessForGenericEntities(entities, callerUGI, entityType);
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
@ -704,6 +710,7 @@ public TimelineEntity getEntity(
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForGenericEntity(entity, callerUGI);
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@ -893,6 +900,7 @@ public TimelineEntity getEntity(
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForGenericEntity(entity, callerUGI);
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@ -956,6 +964,8 @@ public TimelineEntity getFlowRun(
if (context == null) {
throw new BadRequestException("Incorrect UID " + uId);
}
// TODO to be removed or modified once ACL story is played
checkAccess(timelineReaderManager, callerUGI, context.getUserId());
context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString());
entity = timelineReaderManager.getEntity(context,
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
@ -1063,12 +1073,16 @@ public TimelineEntity getFlowRun(
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
try {
entity = timelineReaderManager.getEntity(
TimelineReaderWebServicesUtils.createTimelineReaderContext(
clusterId, userId, flowName, flowRunId, null,
TimelineEntityType.YARN_FLOW_RUN.toString(), null, null),
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
null, metricsToRetrieve, null, null, null, null));
TimelineReaderContext context = TimelineReaderWebServicesUtils
.createTimelineReaderContext(clusterId, userId, flowName, flowRunId,
null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null);
// TODO to be removed or modified once ACL story is played
checkAccess(timelineReaderManager, callerUGI, context.getUserId());
entity = timelineReaderManager.getEntity(context,
TimelineReaderWebServicesUtils
.createTimelineDataToRetrieve(null, metricsToRetrieve, null, null,
null, null));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@ -1156,6 +1170,8 @@ public Set<TimelineEntity> getFlowRuns(
if (context == null) {
throw new BadRequestException("Incorrect UID " + uId);
}
// TODO to be removed or modified once ACL story is played
checkAccess(timelineReaderManager, callerUGI, context.getUserId());
context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString());
entities = timelineReaderManager.getEntities(context,
TimelineReaderWebServicesUtils.createTimelineEntityFilters(
@ -1304,15 +1320,21 @@ public Set<TimelineEntity> getFlowRuns(
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
try {
entities = timelineReaderManager.getEntities(
TimelineReaderWebServicesUtils.createTimelineReaderContext(
clusterId, userId, flowName, null, null,
TimelineEntityType.YARN_FLOW_RUN.toString(), null, null),
TimelineReaderWebServicesUtils.createTimelineEntityFilters(
limit, createdTimeStart, createdTimeEnd, null, null, null,
null, null, null, fromId),
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
null, metricsToRetrieve, fields, null, null, null));
TimelineReaderContext timelineReaderContext = TimelineReaderWebServicesUtils
.createTimelineReaderContext(clusterId, userId, flowName, null,
null, TimelineEntityType.YARN_FLOW_RUN.toString(), null,
null);
// TODO to be removed or modified once ACL story is played
checkAccess(timelineReaderManager, callerUGI,
timelineReaderContext.getUserId());
entities = timelineReaderManager.getEntities(timelineReaderContext,
TimelineReaderWebServicesUtils
.createTimelineEntityFilters(limit, createdTimeStart,
createdTimeEnd, null, null, null, null, null, null, fromId),
TimelineReaderWebServicesUtils
.createTimelineDataToRetrieve(null, metricsToRetrieve, fields,
null, null, null));
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or fromId");
@ -1435,7 +1457,6 @@ public Set<TimelineEntity> getFlows(
long startTime = Time.monotonicNow();
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Configuration config = timelineReaderManager.getConfig();
Set<TimelineEntity> entities = null;
try {
DateRange range = parseDateRange(dateRange);
@ -1455,19 +1476,9 @@ public Set<TimelineEntity> getFlows(
long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
} else if (isDisplayEntityPerUserFilterEnabled(config)) {
Set<TimelineEntity> userEntities = new LinkedHashSet<>();
userEntities.addAll(entities);
for (TimelineEntity entity : userEntities) {
if (entity.getInfo() != null) {
String userId =
(String) entity.getInfo().get(FlowActivityEntity.USER_INFO_KEY);
if (!validateAuthUserWithEntityUser(timelineReaderManager, callerUGI,
userId)) {
entities.remove(entity);
}
}
}
} else {
checkAccess(timelineReaderManager, callerUGI, entities,
FlowActivityEntity.USER_INFO_KEY, true);
}
LOG.info("Processed URL " + url +
" (Took " + (endTime - startTime) + " ms.)");
@ -1552,6 +1563,7 @@ public TimelineEntity getApp(
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForAppEntity(entity, callerUGI);
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@ -1722,6 +1734,7 @@ public TimelineEntity getApp(
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForAppEntity(entity, callerUGI);
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@ -1852,6 +1865,8 @@ public Set<TimelineEntity> getFlowRunApps(
if (context == null) {
throw new BadRequestException("Incorrect UID " + uId);
}
// TODO to be removed or modified once ACL story is played
checkAccess(timelineReaderManager, callerUGI, context.getUserId());
context.setEntityType(TimelineEntityType.YARN_APPLICATION.toString());
entities = timelineReaderManager.getEntities(context,
TimelineReaderWebServicesUtils.createTimelineEntityFilters(
@ -3343,6 +3358,7 @@ public Set<TimelineEntity> getSubAppEntities(
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForSubAppEntities(entities,callerUGI);
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit");
@ -3410,6 +3426,7 @@ public Set<TimelineEntity> getSubAppEntities(@Context HttpServletRequest req,
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForSubAppEntities(entities,callerUGI);
} catch (Exception e) {
handleException(e, url, startTime, "");
}
@ -3422,7 +3439,7 @@ public Set<TimelineEntity> getSubAppEntities(@Context HttpServletRequest req,
return entities;
}
private boolean isDisplayEntityPerUserFilterEnabled(Configuration config) {
static boolean isDisplayEntityPerUserFilterEnabled(Configuration config) {
return !config
.getBoolean(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_READ_AUTH_ENABLED)
@ -3430,8 +3447,76 @@ private boolean isDisplayEntityPerUserFilterEnabled(Configuration config) {
.getBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, false);
}
// TODO to be removed or modified once ACL story is played
private void checkAccessForSubAppEntities(Set<TimelineEntity> entities,
UserGroupInformation callerUGI) throws Exception {
if (entities != null && entities.size() > 0
&& isDisplayEntityPerUserFilterEnabled(
getTimelineReaderManager().getConfig())) {
TimelineReaderContext timelineReaderContext = null;
TimelineEntity entity = entities.iterator().next();
String fromId =
(String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY);
timelineReaderContext =
TimelineFromIdConverter.SUB_APPLICATION_ENTITY_FROMID
.decodeUID(fromId);
checkAccess(getTimelineReaderManager(), callerUGI,
timelineReaderContext.getDoAsUser());
}
}
// TODO to be removed or modified once ACL story is played
private void checkAccessForAppEntity(TimelineEntity entity,
UserGroupInformation callerUGI) throws Exception {
if (entity != null && isDisplayEntityPerUserFilterEnabled(
getTimelineReaderManager().getConfig())) {
String fromId =
(String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY);
TimelineReaderContext timelineReaderContext =
TimelineFromIdConverter.APPLICATION_FROMID.decodeUID(fromId);
checkAccess(getTimelineReaderManager(), callerUGI,
timelineReaderContext.getUserId());
}
}
// TODO to be removed or modified once ACL story is played
private void checkAccessForGenericEntity(TimelineEntity entity,
UserGroupInformation callerUGI) throws Exception {
if (entity != null && isDisplayEntityPerUserFilterEnabled(
getTimelineReaderManager().getConfig())) {
String fromId =
(String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY);
TimelineReaderContext timelineReaderContext =
TimelineFromIdConverter.GENERIC_ENTITY_FROMID.decodeUID(fromId);
checkAccess(getTimelineReaderManager(), callerUGI,
timelineReaderContext.getUserId());
}
}
// TODO to be removed or modified once ACL story is played
private void checkAccessForGenericEntities(Set<TimelineEntity> entities,
UserGroupInformation callerUGI, String entityType) throws Exception {
if (entities != null && entities.size() > 0
&& isDisplayEntityPerUserFilterEnabled(
getTimelineReaderManager().getConfig())) {
TimelineReaderContext timelineReaderContext = null;
TimelineEntity entity = entities.iterator().next();
String uid =
(String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY);
if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
timelineReaderContext =
TimelineFromIdConverter.APPLICATION_FROMID.decodeUID(uid);
} else {
timelineReaderContext =
TimelineFromIdConverter.GENERIC_ENTITY_FROMID.decodeUID(uid);
}
checkAccess(getTimelineReaderManager(), callerUGI,
timelineReaderContext.getUserId());
}
}
// TODO to be removed/modified once ACL story has played
private boolean validateAuthUserWithEntityUser(
static boolean validateAuthUserWithEntityUser(
TimelineReaderManager readerManager, UserGroupInformation ugi,
String entityUser) {
String authUser = TimelineReaderWebServicesUtils.getUserName(ugi);
@ -3442,4 +3527,41 @@ private boolean validateAuthUserWithEntityUser(
}
return (readerManager.checkAccess(ugi) || authUser.equals(requestedUser));
}
// TODO to be removed/modified once ACL story has played
static boolean checkAccess(TimelineReaderManager readerManager,
UserGroupInformation ugi, String entityUser) {
if (isDisplayEntityPerUserFilterEnabled(readerManager.getConfig())) {
if (!validateAuthUserWithEntityUser(readerManager, ugi, entityUser)) {
String userName = ugi.getShortUserName();
String msg = "User " + userName
+ " is not allowed to read TimelineService V2 data.";
throw new ForbiddenException(msg);
}
}
return true;
}
// TODO to be removed or modified once ACL story is played
static void checkAccess(TimelineReaderManager readerManager,
UserGroupInformation callerUGI, Set<TimelineEntity> entities,
String entityUserKey, boolean verifyForAllEntity) {
if (entities.size() > 0 && isDisplayEntityPerUserFilterEnabled(
readerManager.getConfig())) {
Set<TimelineEntity> userEntities = new LinkedHashSet<>();
userEntities.addAll(entities);
for (TimelineEntity entity : userEntities) {
if (entity.getInfo() != null) {
String userId = (String) entity.getInfo().get(entityUserKey);
if (!validateAuthUserWithEntityUser(readerManager, callerUGI,
userId)) {
entities.remove(entity);
if (!verifyForAllEntity) {
break;
}
}
}
}
}
}
}

View File

@ -0,0 +1,154 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.timelineservice.reader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.LinkedHashSet;
import java.util.Set;
public class TestTimelineReaderWebServicesBasicAcl {
private TimelineReaderManager manager;
private static String adminUser = "admin";
private static UserGroupInformation adminUgi =
UserGroupInformation.createRemoteUser(adminUser);
private Configuration config;
@Before public void setUp() throws Exception {
config = new YarnConfiguration();
}
@After public void tearDown() throws Exception {
if (manager != null) {
manager.stop();
manager = null;
}
config = null;
}
@Test public void testTimelineReaderManagerAclsWhenDisabled()
throws Exception {
config.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
config.set(YarnConfiguration.YARN_ADMIN_ACL, adminUser);
manager = new TimelineReaderManager(null);
manager.init(config);
manager.start();
// when acls are disabled, always return true
Assert.assertTrue(manager.checkAccess(null));
// filter is disabled, so should return false
Assert.assertFalse(
TimelineReaderWebServices.isDisplayEntityPerUserFilterEnabled(config));
}
@Test public void testTimelineReaderManagerAclsWhenEnabled()
throws Exception {
Configuration config = new YarnConfiguration();
config.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
config.setBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, true);
config.set(YarnConfiguration.YARN_ADMIN_ACL, adminUser);
manager = new TimelineReaderManager(null);
manager.init(config);
manager.start();
String user1 = "user1";
String user2 = "user2";
UserGroupInformation user1Ugi =
UserGroupInformation.createRemoteUser(user1);
UserGroupInformation user2Ugi =
UserGroupInformation.createRemoteUser(user2);
// false because ugi is null
Assert.assertFalse(TimelineReaderWebServices
.validateAuthUserWithEntityUser(manager, null, user1));
// incoming ugi is admin asking for entity owner user1
Assert.assertTrue(
TimelineReaderWebServices.checkAccess(manager, adminUgi, user1));
// incoming ugi is admin asking for entity owner user1
Assert.assertTrue(
TimelineReaderWebServices.checkAccess(manager, adminUgi, user2));
// incoming ugi is non-admin i.e user1Ugi asking for entity owner user2
try {
TimelineReaderWebServices.checkAccess(manager, user1Ugi, user2);
Assert.fail("user1Ugi is not allowed to view user2");
} catch (ForbiddenException e) {
// expected
}
// incoming ugi is non-admin i.e user2Ugi asking for entity owner user1
try {
TimelineReaderWebServices.checkAccess(manager, user1Ugi, user2);
Assert.fail("user2Ugi is not allowed to view user1");
} catch (ForbiddenException e) {
// expected
}
String userKey = "user";
// incoming ugi is admin asking for entities
Set<TimelineEntity> entities = createEntities(10, userKey);
TimelineReaderWebServices
.checkAccess(manager, adminUgi, entities, userKey, true);
// admin is allowed to view other entities
Assert.assertTrue(entities.size() == 10);
// incoming ugi is user1Ugi asking for entities
// only user1 entities are allowed to view
entities = createEntities(5, userKey);
TimelineReaderWebServices
.checkAccess(manager, user1Ugi, entities, userKey, true);
Assert.assertTrue(entities.size() == 1);
Assert
.assertEquals(user1, entities.iterator().next().getInfo().get(userKey));
// incoming ugi is user2Ugi asking for entities
// only user2 entities are allowed to view
entities = createEntities(8, userKey);
TimelineReaderWebServices
.checkAccess(manager, user2Ugi, entities, userKey, true);
Assert.assertTrue(entities.size() == 1);
Assert
.assertEquals(user2, entities.iterator().next().getInfo().get(userKey));
}
Set<TimelineEntity> createEntities(int noOfUsers, String userKey) {
Set<TimelineEntity> entities = new LinkedHashSet<>();
for (int i = 0; i < noOfUsers; i++) {
TimelineEntity e = new TimelineEntity();
e.setType("user" + i);
e.setId("user" + i);
e.getInfo().put(userKey, "user" + i);
entities.add(e);
}
return entities;
}
}