big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-4522. Queue acl can be checked at app submission. (Jian He via wangda)

Browse Source
This commit is contained in:
Wangda Tan 2015-12-30 15:30:12 -08:00
parent ab725cff66
commit 8310b2e9ff
9 changed files with 43 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
View File

@ -261,7 +261,7 @@ private void submitApp()
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
ugi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws YarnException {
public Object run() throws YarnException, IOException {
rm.getClientRMService().submitApplication(subAppRequest);
return null;
}

2
hadoop-yarn-project/CHANGES.txt
View File

@ -82,6 +82,8 @@ Release 2.9.0 - UNRELEASED
YARN-3480. Remove attempts that are beyond max-attempt limit from state
store. (Jun Gong via jianhe)
YARN-4522. Queue acl can be checked at app submission. (Jian He via wangda)
OPTIMIZATIONS
BUG FIXES

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
View File

@ -551,7 +551,7 @@ public GetContainersResponse getContainers(GetContainersRequest request)
@Override
public SubmitApplicationResponse submitApplication(
SubmitApplicationRequest request) throws YarnException {
SubmitApplicationRequest request) throws YarnException, IOException {
resetStartFailoverFlag(true);
// make sure failover has been triggered

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -549,7 +549,7 @@ public GetContainersResponse getContainers(GetContainersRequest request)
@Override
public SubmitApplicationResponse submitApplication(
SubmitApplicationRequest request) throws YarnException {
SubmitApplicationRequest request) throws YarnException, IOException {
ApplicationSubmissionContext submissionContext = request
.getApplicationSubmissionContext();
ApplicationId applicationId = submissionContext.getApplicationId();

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
View File

@ -26,6 +26,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputByteBuffer;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
@ -33,6 +34,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
@ -55,6 +57,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@ -78,7 +81,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
private final YarnScheduler scheduler;
private final ApplicationACLsManager applicationACLsManager;
private Configuration conf;
private boolean isAclEnabled = false;
public RMAppManager(RMContext context,
YarnScheduler scheduler, ApplicationMasterService masterService,
ApplicationACLsManager applicationACLsManager, Configuration conf) {
@ -97,6 +100,8 @@ public RMAppManager(RMContext context,
if (this.maxCompletedAppsInStateStore > this.maxCompletedAppsInMemory) {
this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory;
}
this.isAclEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE,
YarnConfiguration.DEFAULT_YARN_ACL_ENABLE);
}
/**
@ -276,7 +281,7 @@ protected synchronized void checkAppNumCompletedLimit() {
@SuppressWarnings("unchecked")
protected void submitApplication(
ApplicationSubmissionContext submissionContext, long submitTime,
String user) throws YarnException {
String user) throws YarnException, AccessControlException {
ApplicationId applicationId = submissionContext.getApplicationId();
RMAppImpl application =
@ -325,7 +330,8 @@ protected void recoverApplication(ApplicationStateData appState,
private RMAppImpl createAndPopulateNewRMApp(
ApplicationSubmissionContext submissionContext, long submitTime,
String user, boolean isRecovery) throws YarnException {
String user, boolean isRecovery)
throws YarnException, AccessControlException {
// Do queue mapping
if (!isRecovery) {
if (rmContext.getQueuePlacementManager() != null) {
@ -346,6 +352,22 @@ private RMAppImpl createAndPopulateNewRMApp(
submissionContext.getQueue(), applicationId);
submissionContext.setPriority(appPriority);
UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
// Since FairScheduler queue mapping is done inside scheduler,
// if FairScheduler is used and the queue doesn't exist, we should not
// fail here because queue will be created inside FS. Ideally, FS queue
// mapping should be done outside scheduler too like CS.
// For now, exclude FS for the acl check.
if (!isRecovery && isAclEnabled && scheduler instanceof CapacityScheduler &&
!scheduler.checkAccess(userUgi, QueueACL.SUBMIT_APPLICATIONS,
submissionContext.getQueue()) &&
!scheduler.checkAccess(userUgi, QueueACL.ADMINISTER_QUEUE,
submissionContext.getQueue())) {
throw new AccessControlException(
"User " + user + " does not have permission to submit "
+ applicationId + " to queue " + submissionContext.getQueue());
}
// Create RMApp
RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf,
submissionContext.getApplicationName(), user,

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
View File

@ -479,14 +479,6 @@ public void submitApplication(ApplicationId applicationId, String userName,
String queue) throws AccessControlException {
// Careful! Locking order is important!
// Check queue ACLs
UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(userName);
if (!hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi)
&& !hasAccess(QueueACL.ADMINISTER_QUEUE, userUgi)) {
throw new AccessControlException("User " + userName + " cannot submit" +
" applications to queue " + getQueuePath());
}
User user = null;
synchronized (this) {

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -29,6 +29,7 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
@ -194,7 +195,7 @@ public int getCompletedAppsInStateStore() {
}
public void submitApplication(
ApplicationSubmissionContext submissionContext, String user)
throws YarnException {
throws YarnException, IOException {
super.submitApplication(submissionContext, System.currentTimeMillis(),
user);
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -893,7 +893,7 @@ public void handle(Event rawEvent) {
public void run() {
try {
rmService.submitApplication(submitRequest1);
} catch (YarnException e) {}
} catch (YarnException | IOException e) {}
}
};
t.start();

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
View File

@ -735,6 +735,16 @@ public void testAppSubmit(String acceptMedia, String contentMedia)
client().addFilter(new LoggingFilter(System.out));
String lrKey = "example";
String queueName = "testqueue";
// create the queue
String[] queues = { "default", "testqueue" };
CapacitySchedulerConfiguration csconf =
new CapacitySchedulerConfiguration();
csconf.setQueues("root", queues);
csconf.setCapacity("root.default", 50.0f);
csconf.setCapacity("root.testqueue", 50.0f);
rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext());
String appName = "test";
String appType = "test-type";
String urlPath = "apps";