big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8319. More YARN pages need to honor yarn.resourcemanager.display.per-user-apps. Contributed by Sunil G.

Browse Source
This commit is contained in:
Rohith Sharma K S 2018-05-24 14:19:46 +05:30
parent 4cc0c9b0ba
commit c05b5d424b
8 changed files with 175 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -121,6 +121,10 @@ private static void addDeprecatedKeys() {
new DeprecationDelta(RM_ZK_RETRY_INTERVAL_MS,
CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS),
});
Configuration.addDeprecations(new DeprecationDelta[] {
new DeprecationDelta("yarn.resourcemanager.display.per-user-apps",
FILTER_ENTITY_LIST_BY_USER)
});
}
//Configurations
@ -3569,11 +3573,16 @@ public static boolean areNodeLabelsEnabled(
public static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS =
NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX + "opts";
/*
/**
* Support to view apps for given user in secure cluster.
* @deprecated This field is deprecated for {@link #FILTER_ENTITY_LIST_BY_USER}
*/
@Deprecated
public static final String DISPLAY_APPS_FOR_LOGGED_IN_USER =
RM_PREFIX + "display.per-user-apps";
public static final String FILTER_ENTITY_LIST_BY_USER =
"yarn.webapp.filter-entity-list-by-user";
public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER =
false;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
View File

@ -182,6 +182,8 @@ public void initializeMemberVariables() {
// Ignore deprecated properties
configurationPrefixToSkipCompare
.add(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS);
configurationPrefixToSkipCompare
.add(YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER);
// Allocate for usage
xmlPropsToSkipCompare = new HashSet<String>();

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -3529,7 +3529,7 @@
</property>
<property>
<name>yarn.resourcemanager.display.per-user-apps</name>
<name>yarn.webapp.filter-entity-list-by-user</name>
<value>false</value>
<description>
Flag to enable display of applications per user as an admin

63
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
View File

@ -22,6 +22,7 @@
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.security.Principal;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@ -54,6 +55,8 @@
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.http.JettyUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -99,6 +102,7 @@ public class NMWebServices {
.getRecordFactory(null);
private final String redirectWSUrl;
private final LogAggregationFileControllerFactory factory;
private boolean filterAppsByUser = false;
private @javax.ws.rs.core.Context
HttpServletRequest request;
@ -119,6 +123,15 @@ public NMWebServices(final Context nm, final ResourceView view,
YarnConfiguration.YARN_LOG_SERVER_WEBSERVICE_URL);
this.factory = new LogAggregationFileControllerFactory(
this.nmContext.getConf());
this.filterAppsByUser = this.nmContext.getConf().getBoolean(
YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
}
public NMWebServices(final Context nm, final ResourceView view,
final WebApp webapp, HttpServletResponse response) {
this(nm, view, webapp);
this.response = response;
}
private void init() {
@ -146,7 +159,8 @@ public NodeInfo getNodeInfo() {
@Path("/apps")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public AppsInfo getNodeApps(@QueryParam("state") String stateQuery,
public AppsInfo getNodeApps(@javax.ws.rs.core.Context HttpServletRequest hsr,
@QueryParam("state") String stateQuery,
@QueryParam("user") String userQuery) {
init();
AppsInfo allApps = new AppsInfo();
@ -169,6 +183,14 @@ public AppsInfo getNodeApps(@QueryParam("state") String stateQuery,
continue;
}
}
// Allow only application-owner/admin for any type of access on the
// application.
if (filterAppsByUser
&& !hasAccess(appInfo.getUser(), entry.getKey(), hsr)) {
continue;
}
allApps.add(appInfo);
}
return allApps;
@ -205,6 +227,16 @@ public ContainersInfo getNodeContainers(@javax.ws.rs.core.Context
}
ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue(),
uriInfo.getBaseUri().toString(), webapp.name(), hsr.getRemoteUser());
ApplicationId appId = entry.getKey().getApplicationAttemptId()
.getApplicationId();
// Allow only application-owner/admin for any type of access on the
// application.
if (filterAppsByUser
&& !hasAccess(entry.getValue().getUser(), appId, hsr)) {
continue;
}
allContainers.add(info);
}
return allContainers;
@ -553,4 +585,33 @@ private Response createRedirectResponse(HttpServletRequest httpRequest,
res.header("Location", redirectPath.toString());
return res.build();
}
protected Boolean hasAccess(String user, ApplicationId appId,
HttpServletRequest hsr) {
// Check for the authorization.
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI != null && !(this.nmContext.getApplicationACLsManager()
.checkAccess(callerUGI, ApplicationAccessType.VIEW_APP, user, appId))) {
return false;
}
return true;
}
private UserGroupInformation getCallerUserGroupInformation(
HttpServletRequest hsr, boolean usePrincipal) {
String remoteUser = hsr.getRemoteUser();
if (usePrincipal) {
Principal princ = hsr.getUserPrincipal();
remoteUser = princ == null ? null : princ.getName();
}
UserGroupInformation callerUGI = null;
if (remoteUser != null) {
callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
}
return callerUGI;
}
}

68
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
View File

@ -22,12 +22,17 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.security.Principal;
import java.util.HashMap;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@ -44,11 +49,13 @@
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppsInfo;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
@ -192,25 +199,28 @@ public void testNodeAppsNone() throws JSONException, Exception {
private HashMap<String, String> addAppContainers(Application app)
throws IOException {
return addAppContainers(app, nmContext);
}
private HashMap<String, String> addAppContainers(Application app,
Context context) throws IOException {
Dispatcher dispatcher = new AsyncDispatcher();
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
app.getAppId(), 1);
ApplicationAttemptId appAttemptId = BuilderUtils
.newApplicationAttemptId(app.getAppId(), 1);
Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
app.getUser(), app.getAppId(), 1);
Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
app.getUser(), app.getAppId(), 2);
nmContext.getContainers()
.put(container1.getContainerId(), container1);
nmContext.getContainers()
.put(container2.getContainerId(), container2);
context.getContainers().put(container1.getContainerId(), container1);
context.getContainers().put(container2.getContainerId(), container2);
app.getContainers().put(container1.getContainerId(), container1);
app.getContainers().put(container2.getContainerId(), container2);
HashMap<String, String> hash = new HashMap<String, String>();
hash.put(container1.getContainerId().toString(), container1
.getContainerId().toString());
hash.put(container2.getContainerId().toString(), container2
.getContainerId().toString());
hash.put(container1.getContainerId().toString(),
container1.getContainerId().toString());
hash.put(container2.getContainerId().toString(),
container2.getContainerId().toString());
return hash;
}
@ -721,4 +731,42 @@ public void verifyNodeAppInfoGeneric(Application app, String id,
user);
}
@Test
public void testNodeAppsUserFiltering() throws JSONException, Exception {
Configuration yarnConf = new Configuration();
yarnConf.setBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, true);
yarnConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
yarnConf.setStrings(YarnConfiguration.YARN_ADMIN_ACL, "admin");
ApplicationACLsManager aclManager = new ApplicationACLsManager(yarnConf);
NMContext context = new NodeManager.NMContext(null, null, dirsHandler,
aclManager, null, false, yarnConf);
Application app = new MockApp(1);
context.getApplications().put(app.getAppId(), app);
addAppContainers(app, context);
Application app2 = new MockApp("foo", 1234, 2);
context.getApplications().put(app2.getAppId(), app2);
addAppContainers(app2, context);
// User "foo" could only see its own apps/containers.
NMWebServices webSvc = new NMWebServices(context, null, nmWebApp,
mock(HttpServletResponse.class));
HttpServletRequest mockHsr = mockHttpServletRequestByUserName("foo");
AppsInfo appsInfo = webSvc.getNodeApps(mockHsr, null, null);
assertEquals(1, appsInfo.getApps().size());
// Admin could see all apps and containers.
HttpServletRequest mockHsrAdmin = mockHttpServletRequestByUserName("admin");
AppsInfo appsInfo2 = webSvc.getNodeApps(mockHsrAdmin, null, null);
assertEquals(2, appsInfo2.getApps().size());
}
private HttpServletRequest mockHttpServletRequestByUserName(String username) {
HttpServletRequest mockHsr = mock(HttpServletRequest.class);
when(mockHsr.getRemoteUser()).thenReturn(username);
Principal principal = mock(Principal.class);
when(principal.getName()).thenReturn(username);
when(mockHsr.getUserPrincipal()).thenReturn(principal);
return mockHsr;
}
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -216,7 +216,7 @@ public class ClientRMService extends AbstractService implements
private ReservationSystem reservationSystem;
private ReservationInputValidator rValidator;
private boolean displayPerUserApps = false;
private boolean filterAppsByUser = false;
private static final EnumSet<RMAppState> ACTIVE_APP_STATES = EnumSet.of(
RMAppState.ACCEPTED, RMAppState.RUNNING);
@ -283,8 +283,8 @@ protected void serviceStart() throws Exception {
refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
this.displayPerUserApps = conf.getBoolean(
YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER,
this.filterAppsByUser = conf.getBoolean(
YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
this.server.start();
@ -922,7 +922,7 @@ public void remove() {
// Given RM is configured to display apps per user, skip apps to which
// this caller doesn't have access to view.
if (displayPerUserApps && !allowAccess) {
if (filterAppsByUser && !allowAccess) {
continue;
}
@ -1840,6 +1840,6 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo(
@VisibleForTesting
public void setDisplayPerUserApps(boolean displayPerUserApps) {
this.displayPerUserApps = displayPerUserApps;
this.filterAppsByUser = displayPerUserApps;
}
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -228,7 +228,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
@VisibleForTesting
boolean isCentralizedNodeLabelConfiguration = true;
private boolean displayPerUserApps = false;
private boolean filterAppsByUser = false;
public final static String DELEGATION_TOKEN_HEADER =
"Hadoop-YARN-RM-Delegation-Token";
@ -241,8 +241,8 @@ public RMWebServices(final ResourceManager rm, Configuration conf) {
this.conf = conf;
isCentralizedNodeLabelConfiguration =
YarnConfiguration.isCentralizedNodeLabelConfiguration(conf);
this.displayPerUserApps = conf.getBoolean(
YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER,
this.filterAppsByUser = conf.getBoolean(
YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
}
@ -654,7 +654,7 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
boolean allowAccess = hasAccess(rmapp, hsr);
// Given RM is configured to display apps per user, skip apps to which
// this caller doesn't have access to view.
if (displayPerUserApps && !allowAccess) {
if (filterAppsByUser && !allowAccess) {
continue;
}

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
View File

@ -23,6 +23,7 @@
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.LinkedHashSet;
import java.util.Locale;
import java.util.Set;
import java.util.TimeZone;
@ -42,12 +43,15 @@
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.JettyUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
@ -1450,6 +1454,19 @@ public Set<TimelineEntity> getFlows(
long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
} else if (isDisplayEntityPerUserFilterEnabled(
timelineReaderManager.getConfig())) {
Set<TimelineEntity> userEntities = new LinkedHashSet<>();
userEntities.addAll(entities);
for (TimelineEntity entity : userEntities) {
if (entity.getInfo() != null) {
String userId =
(String) entity.getInfo().get(FlowActivityEntity.USER_INFO_KEY);
if (!validateAuthUserWithEntityUser(callerUGI, userId)) {
entities.remove(entity);
}
}
}
}
LOG.info("Processed URL " + url +
" (Took " + (endTime - startTime) + " ms.)");
@ -3403,4 +3420,20 @@ public Set<TimelineEntity> getSubAppEntities(@Context HttpServletRequest req,
"Processed URL " + url + " (Took " + (endTime - startTime) + " ms.)");
return entities;
}
private boolean isDisplayEntityPerUserFilterEnabled(Configuration config) {
return config
.getBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, false);
}
private boolean validateAuthUserWithEntityUser(UserGroupInformation ugi,
String entityUser) {
String authUser = TimelineReaderWebServicesUtils.getUserName(ugi);
String requestedUser = TimelineReaderWebServicesUtils.parseStr(entityUser);
if (LOG.isDebugEnabled()) {
LOG.debug(
"Authenticated User: " + authUser + " Requested User:" + entityUser);
}
return authUser.equals(requestedUser);
}
}