diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index ea10ed05d0..3f2df0a900 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -198,6 +198,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -2083,7 +2084,7 @@ protected AppMasterLauncher setupAppMasterLauncher(String clustername, // add the tags if available Set applicationTags = provider.getApplicationTags(sliderFileSystem, - appOperations); + appOperations, clustername); Credentials credentials = null; if (clusterSecure) { @@ -3031,9 +3032,10 @@ public int actionExists(String name, ActionExistsArgs args) throws YarnException appstate.ordinal() < YarnApplicationState.FINISHED.ordinal(); } else { // scan for instance in single --state state - List userInstances = yarnClient.listDeployedInstances(""); state = state.toUpperCase(Locale.ENGLISH); YarnApplicationState desiredState = extractYarnApplicationState(state); + List userInstances = yarnClient + .listDeployedInstances("", EnumSet.of(desiredState), name); ApplicationReport foundInstance = yarnClient.findAppInInstanceList(userInstances, name, desiredState); if (foundInstance != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index d471cdb27c..258ef31b00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -53,6 +53,8 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -115,10 +117,59 @@ public List listInstances(String user) */ public List listDeployedInstances(String user) throws YarnException, IOException { + return listDeployedInstances(user, null); + } + + /** + * List Slider deployedinstances belonging to a specific user in a + * given set of states. + *

+ * Deployed means: known about in the YARN cluster; it will include all apps + * in the specified set of states. + * + * @param user + * user: "" means all users + * @param appStates + * filter by a set of YarnApplicationState + * @return a possibly empty list of Slider AMs + * @throws YarnException + * @throws IOException + */ + public List listDeployedInstances(String user, + EnumSet appStates) + throws YarnException, IOException { + return listDeployedInstances(user, appStates, null); + } + + /** + * List Slider deployedinstances belonging to a specific user in a + * given set of states and filtered by an application name tag. + *

+ * Deployed means: known about in the YARN cluster; it will include all apps + * in the specified set of states and tagged with the specified app name. + * + * @param user + * user: "" means all users + * @param appStates + * filter by a set of YarnApplicationState + * @param appname + * an application name tag in the format defined by + * {@link SliderUtils#createNameTag(String)} + * @return a possibly empty list of Slider AMs + * @throws YarnException + * @throws IOException + */ + public List listDeployedInstances(String user, + EnumSet appStates, String appname) + throws YarnException, IOException { Preconditions.checkArgument(user != null, "Null User"); Set types = new HashSet<>(1); types.add(SliderKeys.APP_TYPE); - List allApps = getApplications(types); + Set tags = null; + if (appname != null) { + tags = Collections.singleton(SliderUtils.createNameTag(appname)); + } + List allApps = getApplications(types, appStates, tags); List results = new ArrayList<>(); for (ApplicationReport report : allApps) { if (StringUtils.isEmpty(user) || user.equals(report.getUser())) { @@ -136,20 +187,11 @@ public List listDeployedInstances(String user) * @param appname application name * @return the list of all matching application instances */ - public List findAllInstances(String user, - String appname) + public List findAllInstances(String user, String appname) throws IOException, YarnException { Preconditions.checkArgument(appname != null, "Null application name"); - List instances = listDeployedInstances(user); - List results = - new ArrayList<>(instances.size()); - for (ApplicationReport report : instances) { - if (report.getName().equals(appname)) { - results.add(report); - } - } - return results; + return listDeployedInstances(user, null, appname); } /** @@ -204,14 +246,12 @@ public void emergencyForceKill(String applicationId) // user wants all instances killed String user = getUsername(); log.info("Killing all applications belonging to {}", user); - Collection instances = listDeployedInstances(user); + Collection instances = listDeployedInstances(user, + SliderUtils.getAllLiveAppStates()); for (ApplicationReport instance : instances) { - if (isApplicationLive(instance)) { - ApplicationId appId = instance.getApplicationId(); - log.info("Killing Application {}", appId); - - killRunningApplication(appId, "forced kill"); - } + ApplicationId appId = instance.getApplicationId(); + log.info("Killing Application {}", appId); + killRunningApplication(appId, "forced kill"); } } else { ApplicationId appId = ConverterUtils.toApplicationId(applicationId); @@ -290,21 +330,11 @@ public ApplicationReport monitorAppToState( * @return the list of all matching application instances */ public List findAllLiveInstances(String user, - String appname) throws - YarnException, - IOException { + String appname) throws YarnException, IOException { Preconditions.checkArgument(StringUtils.isNotEmpty(appname), "Null/empty application name"); - List instances = listDeployedInstances(user); - List results = - new ArrayList(instances.size()); - for (ApplicationReport app : instances) { - if (app.getName().equals(appname) - && isApplicationLive(app)) { - results.add(app); - } - } - return results; + return listDeployedInstances(user, SliderUtils.getAllLiveAppStates(), + appname); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index b101d34349..4457957346 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -104,6 +104,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.Date; +import java.util.EnumSet; import java.util.Enumeration; import java.util.HashMap; import java.util.List; @@ -2656,4 +2657,38 @@ public static String trimPrefix(String prefix) { } return prefix; } -} + + public static String createNameTag(String name) { + return "Name: " + name; + } + + public static String createVersionTag(String version) { + return "Version: " + version; + } + + public static String createDescriptionTag(String description) { + return "Description: " + description; + } + + /** + * Get all YarnApplicationState values which signify that an application is + * in RUNNING or pre-RUNNING state. + * + * @return all live app states + */ + public static EnumSet getAllLiveAppStates() { + return EnumSet.range(YarnApplicationState.NEW, + YarnApplicationState.RUNNING); + } + + /** + * Get all YarnApplicationState values which signify that an application is + * not live, which means it is in one of the post RUNNING states. + * + * @return all non live app states + */ + public static EnumSet getAllNonLiveAppStates() { + return EnumSet.range(YarnApplicationState.FINISHED, + YarnApplicationState.KILLED); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java index 1bdfb9c181..71cc193454 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java @@ -86,7 +86,7 @@ public List findAllLiveInstances(String appname) public ApplicationReport findInstance(String appname) throws YarnException, IOException { - List instances = listInstances(null); + List instances = listInstances(null, appname); return yarnClient.findClusterInInstanceList(instances, appname); } @@ -107,8 +107,25 @@ public List listInstances() */ public List listInstances(String user) throws YarnException, IOException { + return listInstances(user, null); + } + + /** + * List all instances belonging to a specific user and a specific appname. + * + * @param user + * user if not the default. null means default, "" means all users, + * otherwise it is the name of a user + * @param appname + * application name set as an application tag + * @return a possibly empty list of AMs + * @throws YarnException + * @throws IOException + */ + public List listInstances(String user, String appname) + throws YarnException, IOException { String listUser = user == null ? username : user; - return yarnClient.listDeployedInstances(listUser); + return yarnClient.listDeployedInstances(listUser, null, appname); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java index f59c347e43..01444fde4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.conf.MapOperations; @@ -37,6 +38,7 @@ import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.HashSet; import java.util.Set; import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES; @@ -217,10 +219,34 @@ public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSys * @return the set of tags. */ public Set getApplicationTags(SliderFileSystem fileSystem, - ConfTreeOperations appConf) throws SliderException { + ConfTreeOperations appConf, String appName) throws SliderException { return Collections.emptySet(); } + /** + * Generates a fixed format of application tags given one or more of + * application name, version and description. This allows subsequent query for + * an application with a name only, version only or description only or any + * combination of those as filters. + * + * @param appName name of the application + * @param appVersion version of the application + * @param appDescription brief description of the application + * @return + */ + public final Set createApplicationTags(String appName, + String appVersion, String appDescription) { + Set tags = new HashSet<>(); + tags.add(SliderUtils.createNameTag(appName)); + if (appVersion != null) { + tags.add(SliderUtils.createVersionTag(appVersion)); + } + if (appDescription != null) { + tags.add(SliderUtils.createDescriptionTag(appDescription)); + } + return tags; + } + /** * Process client operations for applications such as install, configure * @param fileSystem diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java index d554427776..86d87acb60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java @@ -34,6 +34,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE; @@ -93,4 +94,11 @@ public void validateInstanceDefinition(AggregateConf instanceDefinition, } } } + + @Override + public Set getApplicationTags(SliderFileSystem fileSystem, + ConfTreeOperations appConf, String appName) throws SliderException { + return createApplicationTags(appName, null, null); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java index adf613cc45..92df0487ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java @@ -68,23 +68,19 @@ public void init() throws IOException { */ @Override public ProbeStatus ping(boolean livePing) { - ProbeStatus status = new ProbeStatus(); try { - - List instances = - yarnClient.listDeployedInstances(username); - ApplicationReport instance = - yarnClient.findClusterInInstanceList(instances, clustername); + List instances = yarnClient + .listDeployedInstances(username, null, clustername); + ApplicationReport instance = yarnClient + .findClusterInInstanceList(instances, clustername); if (null == instance) { throw UnknownApplicationInstanceException.unknownInstance(clustername); } - status.succeed(this); } catch (Exception e) { status.fail(this, e); } return status; - } }