big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-5883 Avoid or eliminate expensive YARN get all applications call. Contributed by Gour Saha

Browse Source
This commit is contained in:
Billie Rinaldi 2016-11-29 13:14:45 -08:00 committed by Jian He
parent 234dba84a1
commit 7b8fd3abd6
7 changed files with 160 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
View File

@ -198,6 +198,7 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@ -2083,7 +2084,7 @@ protected AppMasterLauncher setupAppMasterLauncher(String clustername,
// add the tags if available
Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
appOperations);
appOperations, clustername);
Credentials credentials = null;
if (clusterSecure) {
@ -3031,9 +3032,10 @@ public int actionExists(String name, ActionExistsArgs args) throws YarnException
appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
} else {
// scan for instance in single --state state
List<ApplicationReport> userInstances = yarnClient.listDeployedInstances("");
state = state.toUpperCase(Locale.ENGLISH);
YarnApplicationState desiredState = extractYarnApplicationState(state);
List<ApplicationReport> userInstances = yarnClient
.listDeployedInstances("", EnumSet.of(desiredState), name);
ApplicationReport foundInstance =
yarnClient.findAppInInstanceList(userInstances, name, desiredState);
if (foundInstance != null) {

94
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
View File

@ -53,6 +53,8 @@
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -115,10 +117,59 @@ public List<ApplicationReport> listInstances(String user)
*/
public List<ApplicationReport> listDeployedInstances(String user)
throws YarnException, IOException {
return listDeployedInstances(user, null);
}
/**
* List Slider <i>deployed</i>instances belonging to a specific user in a
* given set of states.
* <p>
* Deployed means: known about in the YARN cluster; it will include all apps
* in the specified set of states.
*
* @param user
* user: "" means all users
* @param appStates
* filter by a set of YarnApplicationState
* @return a possibly empty list of Slider AMs
* @throws YarnException
* @throws IOException
*/
public List<ApplicationReport> listDeployedInstances(String user,
EnumSet<YarnApplicationState> appStates)
throws YarnException, IOException {
return listDeployedInstances(user, appStates, null);
}
/**
* List Slider <i>deployed</i>instances belonging to a specific user in a
* given set of states and filtered by an application name tag.
* <p>
* Deployed means: known about in the YARN cluster; it will include all apps
* in the specified set of states and tagged with the specified app name.
*
* @param user
* user: "" means all users
* @param appStates
* filter by a set of YarnApplicationState
* @param appname
* an application name tag in the format defined by
* {@link SliderUtils#createNameTag(String)}
* @return a possibly empty list of Slider AMs
* @throws YarnException
* @throws IOException
*/
public List<ApplicationReport> listDeployedInstances(String user,
EnumSet<YarnApplicationState> appStates, String appname)
throws YarnException, IOException {
Preconditions.checkArgument(user != null, "Null User");
Set<String> types = new HashSet<>(1);
types.add(SliderKeys.APP_TYPE);
List<ApplicationReport> allApps = getApplications(types);
Set<String> tags = null;
if (appname != null) {
tags = Collections.singleton(SliderUtils.createNameTag(appname));
}
List<ApplicationReport> allApps = getApplications(types, appStates, tags);
List<ApplicationReport> results = new ArrayList<>();
for (ApplicationReport report : allApps) {
if (StringUtils.isEmpty(user) || user.equals(report.getUser())) {
@ -136,20 +187,11 @@ public List<ApplicationReport> listDeployedInstances(String user)
* @param appname application name
* @return the list of all matching application instances
*/
public List<ApplicationReport> findAllInstances(String user,
String appname)
public List<ApplicationReport> findAllInstances(String user, String appname)
throws IOException, YarnException {
Preconditions.checkArgument(appname != null, "Null application name");
List<ApplicationReport> instances = listDeployedInstances(user);
List<ApplicationReport> results =
new ArrayList<>(instances.size());
for (ApplicationReport report : instances) {
if (report.getName().equals(appname)) {
results.add(report);
}
}
return results;
return listDeployedInstances(user, null, appname);
}
/**
@ -204,14 +246,12 @@ public void emergencyForceKill(String applicationId)
// user wants all instances killed
String user = getUsername();
log.info("Killing all applications belonging to {}", user);
Collection<ApplicationReport> instances = listDeployedInstances(user);
Collection<ApplicationReport> instances = listDeployedInstances(user,
SliderUtils.getAllLiveAppStates());
for (ApplicationReport instance : instances) {
if (isApplicationLive(instance)) {
ApplicationId appId = instance.getApplicationId();
log.info("Killing Application {}", appId);
killRunningApplication(appId, "forced kill");
}
ApplicationId appId = instance.getApplicationId();
log.info("Killing Application {}", appId);
killRunningApplication(appId, "forced kill");
}
} else {
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
@ -290,21 +330,11 @@ public ApplicationReport monitorAppToState(
* @return the list of all matching application instances
*/
public List<ApplicationReport> findAllLiveInstances(String user,
String appname) throws
YarnException,
IOException {
String appname) throws YarnException, IOException {
Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
"Null/empty application name");
List<ApplicationReport> instances = listDeployedInstances(user);
List<ApplicationReport> results =
new ArrayList<ApplicationReport>(instances.size());
for (ApplicationReport app : instances) {
if (app.getName().equals(appname)
&& isApplicationLive(app)) {
results.add(app);
}
}
return results;
return listDeployedInstances(user, SliderUtils.getAllLiveAppStates(),
appname);
}
/**

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
View File

@ -104,6 +104,7 @@
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.EnumSet;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
@ -2656,4 +2657,38 @@ public static String trimPrefix(String prefix) {
}
return prefix;
}
}
public static String createNameTag(String name) {
return "Name: " + name;
}
public static String createVersionTag(String version) {
return "Version: " + version;
}
public static String createDescriptionTag(String description) {
return "Description: " + description;
}
/**
* Get all YarnApplicationState values which signify that an application is
* in RUNNING or pre-RUNNING state.
*
* @return all live app states
*/
public static EnumSet<YarnApplicationState> getAllLiveAppStates() {
return EnumSet.range(YarnApplicationState.NEW,
YarnApplicationState.RUNNING);
}
/**
* Get all YarnApplicationState values which signify that an application is
* not live, which means it is in one of the post RUNNING states.
*
* @return all non live app states
*/
public static EnumSet<YarnApplicationState> getAllNonLiveAppStates() {
return EnumSet.range(YarnApplicationState.FINISHED,
YarnApplicationState.KILLED);
}
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
View File

@ -86,7 +86,7 @@ public List<ApplicationReport> findAllLiveInstances(String appname)
public ApplicationReport findInstance(String appname) throws
YarnException,
IOException {
List<ApplicationReport> instances = listInstances(null);
List<ApplicationReport> instances = listInstances(null, appname);
return yarnClient.findClusterInInstanceList(instances, appname);
}
@ -107,8 +107,25 @@ public List<ApplicationReport> listInstances()
*/
public List<ApplicationReport> listInstances(String user)
throws YarnException, IOException {
return listInstances(user, null);
}
/**
* List all instances belonging to a specific user and a specific appname.
*
* @param user
* user if not the default. null means default, "" means all users,
* otherwise it is the name of a user
* @param appname
* application name set as an application tag
* @return a possibly empty list of AMs
* @throws YarnException
* @throws IOException
*/
public List<ApplicationReport> listInstances(String user, String appname)
throws YarnException, IOException {
String listUser = user == null ? username : user;
return yarnClient.listDeployedInstances(listUser);
return yarnClient.listDeployedInstances(listUser, null, appname);
}
/**

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
View File

@ -23,6 +23,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.conf.MapOperations;
@ -37,6 +38,7 @@
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.HashSet;
import java.util.Set;
import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
@ -217,10 +219,34 @@ public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSys
* @return the set of tags.
*/
public Set<String> getApplicationTags(SliderFileSystem fileSystem,
ConfTreeOperations appConf) throws SliderException {
ConfTreeOperations appConf, String appName) throws SliderException {
return Collections.emptySet();
}
/**
* Generates a fixed format of application tags given one or more of
* application name, version and description. This allows subsequent query for
* an application with a name only, version only or description only or any
* combination of those as filters.
*
* @param appName name of the application
* @param appVersion version of the application
* @param appDescription brief description of the application
* @return
*/
public final Set<String> createApplicationTags(String appName,
String appVersion, String appDescription) {
Set<String> tags = new HashSet<>();
tags.add(SliderUtils.createNameTag(appName));
if (appVersion != null) {
tags.add(SliderUtils.createVersionTag(appVersion));
}
if (appDescription != null) {
tags.add(SliderUtils.createDescriptionTag(appDescription));
}
return tags;
}
/**
* Process client operations for applications such as install, configure
* @param fileSystem

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
View File

@ -34,6 +34,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE;
@ -93,4 +94,11 @@ public void validateInstanceDefinition(AggregateConf instanceDefinition,
}
}
}
@Override
public Set<String> getApplicationTags(SliderFileSystem fileSystem,
ConfTreeOperations appConf, String appName) throws SliderException {
return createApplicationTags(appName, null, null);
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
View File

@ -68,23 +68,19 @@ public void init() throws IOException {
*/
@Override
public ProbeStatus ping(boolean livePing) {
ProbeStatus status = new ProbeStatus();
try {
List<ApplicationReport> instances =
yarnClient.listDeployedInstances(username);
ApplicationReport instance =
yarnClient.findClusterInInstanceList(instances, clustername);
List<ApplicationReport> instances = yarnClient
.listDeployedInstances(username, null, clustername);
ApplicationReport instance = yarnClient
.findClusterInInstanceList(instances, clustername);
if (null == instance) {
throw UnknownApplicationInstanceException.unknownInstance(clustername);
}
status.succeed(this);
} catch (Exception e) {
status.fail(this, e);
}
return status;
}
}