big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7157. Add admin configuration to filter per-user's apps in secure cluster. Contributed by Sunil G.

Browse Source
This commit is contained in:
Rohith Sharma K S 2017-09-13 23:36:39 +05:30
parent 8277fab2be
commit 5324388cf2
4 changed files with 110 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -3158,6 +3158,14 @@ public static boolean areNodeLabelsEnabled(
public static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS =
NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX + "opts";
/*
* Support to view apps for given user in secure cluster.
*/
public static final String DISPLAY_APPS_FOR_LOGGED_IN_USER =
RM_PREFIX + "display.per-user-apps";
public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER =
false;
// RM and NM CSRF props
public static final String REST_CSRF = "webapp.rest-csrf.";
public static final String RM_CSRF_PREFIX = RM_PREFIX + REST_CSRF;

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -3323,4 +3323,14 @@
to specify details about the individual resource types.
</description>
</property>
<property>
<name>yarn.resourcemanager.display.per-user-apps</name>
<value>false</value>
<description>
Flag to enable display of applications per user as an admin
configuration.
</description>
</property>
</configuration>

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -215,6 +215,8 @@ public class ClientRMService extends AbstractService implements
private ReservationSystem reservationSystem;
private ReservationInputValidator rValidator;
private boolean displayPerUserApps = false;
private static final EnumSet<RMAppState> ACTIVE_APP_STATES = EnumSet.of(
RMAppState.ACCEPTED, RMAppState.RUNNING);
@ -275,7 +277,11 @@ protected void serviceStart() throws Exception {
}
refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
this.displayPerUserApps = conf.getBoolean(
YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER,
YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
this.server.start();
clientBindAddress = conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST,
YarnConfiguration.RM_ADDRESS,
@ -909,6 +915,12 @@ public void remove() {
continue;
}
// Given RM is configured to display apps per user, skip apps to which
// this caller doesn't have access to view.
if (displayPerUserApps && !allowAccess) {
continue;
}
reports.add(application.createAndGetApplicationReport(
callerUGI.getUserName(), allowAccess));
}
@ -1804,4 +1816,9 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo(
response.setResourceTypeInfo(ResourceUtils.getResourcesTypeInfo());
return response;
}
@VisibleForTesting
public void setDisplayPerUserApps(boolean displayPerUserApps) {
this.displayPerUserApps = displayPerUserApps;
}
}

74
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -1121,6 +1121,12 @@ public void handle(Event event) {}
assertEquals("Incorrect number of applications for user", 3,
rmService.getApplications(request).getApplicationList().size());
rmService.setDisplayPerUserApps(true);
userSet.clear();
assertEquals("Incorrect number of applications for user", 6,
rmService.getApplications(request).getApplicationList().size());
rmService.setDisplayPerUserApps(false);
// Check tags
request = GetApplicationsRequest.newInstance(
ApplicationsRequestScope.ALL, null, null, null, null, null, null,
@ -2107,4 +2113,72 @@ protected ClientRMService createClientRMService() {
rm.stop();
rpc.stopProxy(client, conf);
}
@Test
public void testGetApplicationsWithPerUserApps()
throws IOException, YarnException {
/*
* Submit 3 applications alternately in two queues
*/
// Basic setup
YarnScheduler yarnScheduler = mockYarnScheduler();
RMContext rmContext = mock(RMContext.class);
mockRMContext(yarnScheduler, rmContext);
RMStateStore stateStore = mock(RMStateStore.class);
when(rmContext.getStateStore()).thenReturn(stateStore);
doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
.getRMTimelineCollectorManager();
RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
mock(ApplicationACLsManager.class), new Configuration());
when(rmContext.getDispatcher().getEventHandler())
.thenReturn(new EventHandler<Event>() {
public void handle(Event event) {
}
});
// Simulate Queue ACL manager which returns false always
QueueACLsManager queueAclsManager = mock(QueueACLsManager.class);
when(queueAclsManager.checkAccess(any(UserGroupInformation.class),
any(QueueACL.class), any(RMApp.class), any(String.class),
anyListOf(String.class))).thenReturn(false);
// Simulate app ACL manager which returns false always
ApplicationACLsManager appAclsManager = mock(ApplicationACLsManager.class);
when(appAclsManager.checkAccess(eq(UserGroupInformation.getCurrentUser()),
any(ApplicationAccessType.class), any(String.class),
any(ApplicationId.class))).thenReturn(false);
ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
appManager, appAclsManager, queueAclsManager, null);
rmService.init(new Configuration());
// Initialize appnames and queues
String[] queues = {QUEUE_1, QUEUE_2};
String[] appNames = {MockApps.newAppName(), MockApps.newAppName(),
MockApps.newAppName()};
ApplicationId[] appIds = {getApplicationId(101), getApplicationId(102),
getApplicationId(103)};
List<String> tags = Arrays.asList("Tag1", "Tag2", "Tag3");
long[] submitTimeMillis = new long[3];
// Submit applications
for (int i = 0; i < appIds.length; i++) {
ApplicationId appId = appIds[i];
SubmitApplicationRequest submitRequest = mockSubmitAppRequest(appId,
appNames[i], queues[i % queues.length],
new HashSet<String>(tags.subList(0, i + 1)));
rmService.submitApplication(submitRequest);
submitTimeMillis[i] = System.currentTimeMillis();
}
// Test different cases of ClientRMService#getApplications()
GetApplicationsRequest request = GetApplicationsRequest.newInstance();
assertEquals("Incorrect total number of apps", 6,
rmService.getApplications(request).getApplicationList().size());
rmService.setDisplayPerUserApps(true);
assertEquals("Incorrect number of applications for user", 0,
rmService.getApplications(request).getApplicationList().size());
rmService.setDisplayPerUserApps(false);
}
}