diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java index 2272e3ed91..d61bf024cc 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java @@ -261,7 +261,7 @@ private void submitApp() UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); ugi.doAs(new PrivilegedExceptionAction() { @Override - public Object run() throws YarnException { + public Object run() throws YarnException, IOException { rm.getClientRMService().submitApplication(subAppRequest); return null; } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b9789b85bf..4c8eecb779 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -82,6 +82,8 @@ Release 2.9.0 - UNRELEASED YARN-3480. Remove attempts that are beyond max-attempt limit from state store. (Jun Gong via jianhe) + YARN-4522. Queue acl can be checked at app submission. (Jian He via wangda) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index 45629b266a..cf7fcc5acb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -551,7 +551,7 @@ public GetContainersResponse getContainers(GetContainersRequest request) @Override public SubmitApplicationResponse submitApplication( - SubmitApplicationRequest request) throws YarnException { + SubmitApplicationRequest request) throws YarnException, IOException { resetStartFailoverFlag(true); // make sure failover has been triggered diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index b29048717c..4722e1cd58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -549,7 +549,7 @@ public GetContainersResponse getContainers(GetContainersRequest request) @Override public SubmitApplicationResponse submitApplication( - SubmitApplicationRequest request) throws YarnException { + SubmitApplicationRequest request) throws YarnException, IOException { ApplicationSubmissionContext submissionContext = request .getApplicationSubmissionContext(); ApplicationId applicationId = submissionContext.getApplicationId(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 711a7a7c0a..4344914427 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataInputByteBuffer; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; @@ -33,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; @@ -55,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -78,7 +81,7 @@ public class RMAppManager implements EventHandler, private final YarnScheduler scheduler; private final ApplicationACLsManager applicationACLsManager; private Configuration conf; - + private boolean isAclEnabled = false; public RMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationACLsManager applicationACLsManager, Configuration conf) { @@ -97,6 +100,8 @@ public RMAppManager(RMContext context, if (this.maxCompletedAppsInStateStore > this.maxCompletedAppsInMemory) { this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory; } + this.isAclEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, + YarnConfiguration.DEFAULT_YARN_ACL_ENABLE); } /** @@ -276,7 +281,7 @@ protected synchronized void checkAppNumCompletedLimit() { @SuppressWarnings("unchecked") protected void submitApplication( ApplicationSubmissionContext submissionContext, long submitTime, - String user) throws YarnException { + String user) throws YarnException, AccessControlException { ApplicationId applicationId = submissionContext.getApplicationId(); RMAppImpl application = @@ -325,7 +330,8 @@ protected void recoverApplication(ApplicationStateData appState, private RMAppImpl createAndPopulateNewRMApp( ApplicationSubmissionContext submissionContext, long submitTime, - String user, boolean isRecovery) throws YarnException { + String user, boolean isRecovery) + throws YarnException, AccessControlException { // Do queue mapping if (!isRecovery) { if (rmContext.getQueuePlacementManager() != null) { @@ -346,6 +352,22 @@ private RMAppImpl createAndPopulateNewRMApp( submissionContext.getQueue(), applicationId); submissionContext.setPriority(appPriority); + UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user); + // Since FairScheduler queue mapping is done inside scheduler, + // if FairScheduler is used and the queue doesn't exist, we should not + // fail here because queue will be created inside FS. Ideally, FS queue + // mapping should be done outside scheduler too like CS. + // For now, exclude FS for the acl check. + if (!isRecovery && isAclEnabled && scheduler instanceof CapacityScheduler && + !scheduler.checkAccess(userUgi, QueueACL.SUBMIT_APPLICATIONS, + submissionContext.getQueue()) && + !scheduler.checkAccess(userUgi, QueueACL.ADMINISTER_QUEUE, + submissionContext.getQueue())) { + throw new AccessControlException( + "User " + user + " does not have permission to submit " + + applicationId + " to queue " + submissionContext.getQueue()); + } + // Create RMApp RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, submissionContext.getApplicationName(), user, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index be1ba89ef0..5c3f4b9d09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -479,14 +479,6 @@ public void submitApplication(ApplicationId applicationId, String userName, String queue) throws AccessControlException { // Careful! Locking order is important! - // Check queue ACLs - UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(userName); - if (!hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi) - && !hasAccess(QueueACL.ADMINISTER_QUEUE, userUgi)) { - throw new AccessControlException("User " + userName + " cannot submit" + - " applications to queue " + getQueuePath()); - } - User user = null; synchronized (this) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index c4356921df..075df47383 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -29,6 +29,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; @@ -194,7 +195,7 @@ public int getCompletedAppsInStateStore() { } public void submitApplication( ApplicationSubmissionContext submissionContext, String user) - throws YarnException { + throws YarnException, IOException { super.submitApplication(submissionContext, System.currentTimeMillis(), user); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 54ebd25bb2..a5b0b680ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -893,7 +893,7 @@ public void handle(Event rawEvent) { public void run() { try { rmService.submitApplication(submitRequest1); - } catch (YarnException e) {} + } catch (YarnException | IOException e) {} } }; t.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index df3c2bc53a..61579f8303 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -735,6 +735,16 @@ public void testAppSubmit(String acceptMedia, String contentMedia) client().addFilter(new LoggingFilter(System.out)); String lrKey = "example"; String queueName = "testqueue"; + + // create the queue + String[] queues = { "default", "testqueue" }; + CapacitySchedulerConfiguration csconf = + new CapacitySchedulerConfiguration(); + csconf.setQueues("root", queues); + csconf.setCapacity("root.default", 50.0f); + csconf.setCapacity("root.testqueue", 50.0f); + rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext()); + String appName = "test"; String appType = "test-type"; String urlPath = "apps";