diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 83b484129a..32d78b4542 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -133,7 +133,6 @@ import org.apache.slider.core.zk.ZKIntegration; import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.ProviderUtils; -import org.apache.slider.providers.SliderProviderFactory; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.rpc.RpcBinder; import org.apache.slider.server.services.utility.AbstractSliderLaunchedService; @@ -257,8 +256,6 @@ public Configuration bindArgs(Configuration config, String... args) throws Excep config = super.bindArgs(config, args); serviceArgs = new ClientArgs(args); serviceArgs.parse(); - // add the slider XML config - ConfigHelper.injectSliderXMLResource(); // yarn-ify YarnConfiguration yarnConfiguration = new YarnConfiguration(config); return patchConfiguration(yarnConfiguration); @@ -622,14 +619,6 @@ public int actionAmSuicide(String clustername, return EXIT_SUCCESS; } - @Override - public AbstractClientProvider createClientProvider(String provider) - throws SliderException { - SliderProviderFactory factory = - SliderProviderFactory.createSliderProviderFactory(provider); - return factory.createClientProvider(); - } - private Application getApplicationFromArgs(String clusterName, AbstractClusterBuildingActionArgs args) throws IOException { File file = args.getAppDef(); @@ -893,7 +882,7 @@ private Path checkAppNotExistOnHdfs(Application application) private void persistApp(Path appDir, Application application) throws IOException, SliderException { - FsPermission appDirPermission = new FsPermission("777"); + FsPermission appDirPermission = new FsPermission("750"); sliderFileSystem.createWithPermissions(appDir, appDirPermission); Path appJson = new Path(appDir, application.getName() + ".json"); jsonSerDeser @@ -1190,17 +1179,18 @@ private int doClientInstall(ActionClientArgs clientInfo) } } + // TODO handle client install // Only INSTALL is supported - AbstractClientProvider - provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE); - provider.processClientOperation(sliderFileSystem, - getRegistryOperations(), - getConfig(), - "INSTALL", - clientInfo.installLocation, - pkgFile, - config, - clientInfo.name); + // ClientProvider + // provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE); + // provider.processClientOperation(sliderFileSystem, + // getRegistryOperations(), + // getConfig(), + // "INSTALL", + // clientInfo.installLocation, + // pkgFile, + // config, + // clientInfo.name); return EXIT_SUCCESS; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index 197a564f5f..7477c05d20 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -42,7 +42,6 @@ import org.apache.slider.common.params.ActionUpgradeArgs; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.providers.AbstractClientProvider; import java.io.IOException; @@ -62,15 +61,6 @@ public interface SliderClientAPI extends Service { int actionAmSuicide(String clustername, ActionAMSuicideArgs args) throws YarnException, IOException; - /** - * Get the provider for this cluster - * @param provider the name of the provider - * @return the provider instance - * @throws SliderException problems building the provider - */ - AbstractClientProvider createClientProvider(String provider) - throws SliderException; - /** * Manage keytabs leveraged by slider * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 968a90bb5e..734fec5af0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -18,8 +18,6 @@ package org.apache.slider.common; -import org.apache.hadoop.yarn.api.ApplicationConstants; - import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -158,12 +156,7 @@ public interface SliderKeys extends SliderXmlConfKeys { */ String HISTORY_FILENAME_GLOB_PATTERN = HISTORY_FILENAME_PREFIX +"*."+ HISTORY_FILENAME_SUFFIX; - /** - * XML resource listing the standard Slider providers - * {@value} - */ - String SLIDER_XML = "org/apache/slider/slider.xml"; - + String CLUSTER_DIRECTORY = "cluster"; String PACKAGE_DIRECTORY = "package"; @@ -311,5 +304,12 @@ public interface SliderKeys extends SliderXmlConfKeys { String APP_RESOURCES = "application.resources"; String APP_RESOURCES_DIR = "app/resources"; - String APP_PACKAGES_DIR = "app/packages"; + String APP_INSTALL_DIR = "app/install"; + + String OUT_FILE = "stdout.txt"; + String ERR_FILE = "stderr.txt"; + + String QUICK_LINKS = "quicklinks"; + + String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java index 0e94a2947e..25debdc2ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java @@ -50,7 +50,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.atomic.AtomicBoolean; /** * Methods to aid in config, both in the Configuration class and @@ -62,11 +61,6 @@ public class ConfigHelper { private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class); - private static AtomicBoolean sliderResourceInjected = - new AtomicBoolean(false); - private static AtomicBoolean sliderResourceInjectionAttempted = - new AtomicBoolean(false); - /** * Dump the (sorted) configuration * @param conf config @@ -614,37 +608,4 @@ public static Configuration resolveConfiguration( public static void registerDeprecatedConfigItems() { } - /** - * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource - * included - * @return a configuration instance - */ - public static Configuration loadSliderConfiguration() { - Configuration conf = new Configuration(); - conf.addResource(SliderKeys.SLIDER_XML); - return conf; - } - - /** - * Inject the {@link SliderKeys#SLIDER_XML} resource - * into the configuration resources of all configurations. - *

- * This operation is idempotent. - *

- * If the resource is not on the classpath, downgrades, rather than - * fails. - * @return true if the resource was found and loaded. - */ - public static synchronized boolean injectSliderXMLResource() { - if (sliderResourceInjectionAttempted.getAndSet(true)) { - return sliderResourceInjected.get(); - } - URL resourceUrl = getResourceUrl(SliderKeys.SLIDER_XML); - if (resourceUrl != null) { - Configuration.addDefaultResource(SliderKeys.SLIDER_XML); - sliderResourceInjected.set(true); - } - return sliderResourceInjected.get(); - } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java index df174f55c8..185dcd4170 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java @@ -19,32 +19,26 @@ package org.apache.slider.providers; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.slider.api.resource.Artifact; +import org.apache.slider.api.resource.ConfigFile; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.SliderException; import org.codehaus.jettison.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; import java.util.HashSet; import java.util.List; import java.util.Set; -public abstract class AbstractClientProvider extends Configured { - private static final Logger log = - LoggerFactory.getLogger(AbstractClientProvider.class); +public abstract class AbstractClientProvider { - public AbstractClientProvider(Configuration conf) { - super(conf); + public AbstractClientProvider() { } - public abstract String getName(); - - public abstract List getRoles(); - /** * Generates a fixed format of application tags given one or more of * application name, version and description. This allows subsequent query for @@ -70,7 +64,29 @@ public static final Set createApplicationTags(String appName, } /** - * Process client operations for applications such as install, configure + * Validate the artifact. + * @param artifact + */ + public abstract void validateArtifact(Artifact artifact, FileSystem + fileSystem) throws IOException; + + protected abstract void validateConfigFile(ConfigFile configFile, FileSystem + fileSystem) throws IOException; + + /** + * Validate the config files. + * @param configFiles config file list + * @param fileSystem file system + */ + public void validateConfigFiles(List configFiles, FileSystem + fileSystem) throws IOException { + for (ConfigFile configFile : configFiles) { + validateConfigFile(configFile, fileSystem); + } + } + + /** + * Process client operations for applications such as install, configure. * @param fileSystem * @param registryOperations * @param configuration diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java new file mode 100644 index 0000000000..b7fa802113 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers; + +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.ContainerState; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.SliderException; +import org.apache.slider.core.launch.CommandLineBuilder; +import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher; +import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; +import java.util.Map.Entry; + +import static org.apache.slider.util.ServiceApiUtil.$; + +public abstract class AbstractProviderService extends AbstractService + implements ProviderService, SliderKeys { + + protected static final Logger log = + LoggerFactory.getLogger(AbstractProviderService.class); + private static final ProviderUtils providerUtils = new ProviderUtils(log); + protected StateAccessForProviders amState; + protected YarnRegistryViewForProviders yarnRegistry; + private ServiceTimelinePublisher serviceTimelinePublisher; + + protected AbstractProviderService(String name) { + super(name); + } + + public abstract void processArtifact(ContainerLauncher launcher, Component + component, SliderFileSystem fileSystem) throws IOException; + + @Override + public void setAMState(StateAccessForProviders stateAccessor) { + this.amState = stateAccessor; + } + + @Override + public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) { + this.yarnRegistry = yarnRegistry; + } + + public void buildContainerLaunchContext(ContainerLauncher launcher, + Application application, Container container, ProviderRole providerRole, + SliderFileSystem fileSystem, RoleInstance roleInstance) + throws IOException, SliderException { + Component component = providerRole.component; + processArtifact(launcher, component, fileSystem); + + // Generate tokens (key-value pair) for config substitution. + // Get pre-defined tokens + Map globalTokens = amState.getGlobalSubstitutionTokens(); + Map tokensForSubstitution = providerUtils + .initCompTokensForSubstitute(roleInstance); + tokensForSubstitution.putAll(globalTokens); + // Set the environment variables in launcher + launcher.putEnv(SliderUtils + .buildEnvMap(component.getConfiguration(), tokensForSubstitution)); + launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$()); + launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); + if (System.getenv(HADOOP_USER_NAME) != null) { + launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME)); + } + launcher.setEnv("LANG", "en_US.UTF-8"); + launcher.setEnv("LC_ALL", "en_US.UTF-8"); + launcher.setEnv("LANGUAGE", "en_US.UTF-8"); + + for (Entry entry : launcher.getEnv().entrySet()) { + tokensForSubstitution.put($(entry.getKey()), entry.getValue()); + } + providerUtils.addComponentHostTokens(tokensForSubstitution, amState); + + // create config file on hdfs and add local resource + providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem, + component, tokensForSubstitution, roleInstance, amState); + + // substitute launch command + String launchCommand = ProviderUtils + .substituteStrWithTokens(component.getLaunchCommand(), + tokensForSubstitution); + CommandLineBuilder operation = new CommandLineBuilder(); + operation.add(launchCommand); + operation.addOutAndErrFiles(OUT_FILE, ERR_FILE); + launcher.addCommand(operation.build()); + + // publish exports + providerUtils + .substituteMapWithTokens(application.getQuicklinks(), tokensForSubstitution); + PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS, + application.getQuicklinks().entrySet()); + amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf); + if (serviceTimelinePublisher != null) { + serviceTimelinePublisher.serviceAttemptUpdated(application); + } + } + + public boolean processContainerStatus(ContainerId containerId, + ContainerStatus status) { + log.debug("Handling container status: {}", status); + if (SliderUtils.isEmpty(status.getIPs()) || + SliderUtils.isUnset(status.getHost())) { + return true; + } + RoleInstance instance = amState.getOwnedContainer(containerId); + if (instance == null) { + // container is completed? + return false; + } + + try { + providerUtils.updateServiceRecord(amState, yarnRegistry, + containerId.toString(), instance.role, status.getIPs(), status.getHost()); + } catch (IOException e) { + // could not write service record to ZK, log and retry + log.warn("Error updating container {} service record in registry, " + + "retrying", containerId, e); + return true; + } + // TODO publish ip and host + org.apache.slider.api.resource.Container container = + instance.providerRole.component.getContainer(containerId.toString()); + if (container != null) { + container.setIp(StringUtils.join(",", status.getIPs())); + container.setHostname(status.getHost()); + container.setState(ContainerState.READY); + } else { + log.warn(containerId + " not found in Application!"); + } + return false; + } + + @Override + public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) { + this.serviceTimelinePublisher = publisher; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java new file mode 100644 index 0000000000..004e81fe95 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.slider.api.resource.Artifact; +import org.apache.slider.api.resource.ConfigFile; + +import java.io.IOException; +import java.nio.file.Paths; + +public class DefaultClientProvider extends AbstractClientProvider { + + public DefaultClientProvider() { + } + + @Override + public void validateArtifact(Artifact artifact, FileSystem fileSystem) { + } + + @Override + protected void validateConfigFile(ConfigFile configFile, FileSystem + fileSystem) throws IOException { + // validate dest_file is not absolute + if (Paths.get(configFile.getDestFile()).isAbsolute()) { + throw new IllegalArgumentException( + "Dest_file must not be absolute path: " + configFile.getDestFile()); + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java new file mode 100644 index 0000000000..09a1423284 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers; + +public final class DefaultProviderFactory extends SliderProviderFactory { + private static final SliderProviderFactory FACTORY = new + DefaultProviderFactory(); + + private DefaultProviderFactory() {} + + private static class Client { + static final AbstractClientProvider PROVIDER = new DefaultClientProvider(); + } + + private static class Server { + static final ProviderService PROVIDER = new DefaultProviderService(); + } + + @Override + public AbstractClientProvider createClientProvider() { + return Client.PROVIDER; + } + + @Override + public ProviderService createServerProvider() { + return Server.PROVIDER; + } + + public static SliderProviderFactory getInstance() { + return FACTORY; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java new file mode 100644 index 0000000000..7f7d209598 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers; + +import org.apache.slider.api.resource.Component; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.providers.AbstractProviderService; + +import java.io.IOException; + +public class DefaultProviderService extends AbstractProviderService { + + protected DefaultProviderService() { + super(DefaultProviderService.class.getSimpleName()); + } + + @Override + public void processArtifact(ContainerLauncher launcher, Component + component, SliderFileSystem fileSystem) throws IOException { + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java index 5dd4a3221a..9c5264347d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java @@ -18,93 +18,60 @@ package org.apache.slider.providers; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.core.exceptions.BadClusterStateException; +import org.apache.slider.api.resource.Artifact; import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.providers.agent.AgentKeys; +import org.apache.slider.providers.docker.DockerProviderFactory; +import org.apache.slider.providers.tarball.TarballProviderFactory; +import org.apache.slider.util.RestApiErrorMessages; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Base class for factories + * Base class for factories. */ -public abstract class SliderProviderFactory extends Configured { - - public static final String DEFAULT_CLUSTER_TYPE = AgentKeys.PROVIDER_AGENT; - +public abstract class SliderProviderFactory { protected static final Logger log = - LoggerFactory.getLogger(SliderProviderFactory.class); - public static final String PROVIDER_NOT_FOUND = - "Unable to find provider of application type %s"; + LoggerFactory.getLogger(SliderProviderFactory.class); - public SliderProviderFactory(Configuration conf) { - super(conf); - } - - protected SliderProviderFactory() { - } + protected SliderProviderFactory() {} public abstract AbstractClientProvider createClientProvider(); public abstract ProviderService createServerProvider(); + public static synchronized ProviderService getProviderService(Artifact + artifact) { + return createSliderProviderFactory(artifact).createServerProvider(); + } + + public static synchronized AbstractClientProvider getClientProvider(Artifact + artifact) { + return createSliderProviderFactory(artifact).createClientProvider(); + } + /** * Create a provider for a specific application - * @param application app - * @return app instance + * @param artifact artifact + * @return provider factory * @throws SliderException on any instantiation problem */ - public static SliderProviderFactory createSliderProviderFactory(String application) throws - SliderException { - Configuration conf = loadSliderConfiguration(); - if (application == null) { - application = DEFAULT_CLUSTER_TYPE; + public static synchronized SliderProviderFactory createSliderProviderFactory( + Artifact artifact) { + if (artifact == null || artifact.getType() == null) { + log.info("Loading service provider type default"); + return DefaultProviderFactory.getInstance(); } - String providerKey = - String.format(SliderXmlConfKeys.KEY_PROVIDER, application); - if (application.contains(".")) { - log.debug("Treating {} as a classname", application); - String name = "classname.key"; - conf.set(name, application); - providerKey = name; + log.info("Loading service provider type {}", artifact.getType()); + switch (artifact.getType()) { + // TODO add handling for custom types? + // TODO handle application + case DOCKER: + return DockerProviderFactory.getInstance(); + case TARBALL: + return TarballProviderFactory.getInstance(); + default: + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_INVALID); } - - Class providerClass; - try { - providerClass = conf.getClass(providerKey, null, SliderProviderFactory.class); - } catch (RuntimeException e) { - throw new BadClusterStateException(e, "Failed to load provider %s: %s", application, e); - } - if (providerClass == null) { - throw new BadClusterStateException(PROVIDER_NOT_FOUND, application); - } - - Exception ex; - try { - SliderProviderFactory providerFactory = providerClass.newInstance(); - providerFactory.setConf(conf); - return providerFactory; - } catch (Exception e) { - ex = e; - } - //by here the operation failed and ex is set to the value - throw new BadClusterStateException(ex, - "Failed to create an instance of %s : %s", - providerClass, - ex); - } - - /** - * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource - * included - * @return a configuration instance - */ - public static Configuration loadSliderConfiguration() { - Configuration conf = new Configuration(); - conf.addResource(SliderKeys.SLIDER_XML); - return conf; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java deleted file mode 100644 index c4228e4932..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -/* - - */ -public interface AgentKeys { - - String AGENT_TAR = "slider-agent.tar.gz"; - String PROVIDER_AGENT = "agent"; - String ROLE_NODE = "echo"; - - /** - * Template stored in the slider classpath -to use if there is - * no site-specific template - * {@value} - */ - String CONF_RESOURCE = "org/apache/slider/providers/agent/conf/"; - /* URL to talk back to Agent Controller*/ - String CONTROLLER_URL = "agent.controller.url"; - /** - * The location of pre-installed agent path. - * This can be also be dynamically computed based on Yarn installation of agent. - */ - String PACKAGE_PATH = "agent.package.root"; - /** - * The location of the script implementing the command. - */ - String SCRIPT_PATH = "agent.script"; - /** - * Execution home for the agent. - */ - String APP_HOME = "app.home"; - String APP_ROOT = "site.global.app_root"; - String APP_CLIENT_ROOT = "client_root"; - /** - * Runas user of the application - */ - String RUNAS_USER = "site.global.app_user"; - /** - * Name of the service. - */ - String SERVICE_NAME = "app.name"; - String ARG_LABEL = "--label"; - String ARG_HOST = "--host"; - String ARG_PORT = "--port"; - String ARG_SECURED_PORT = "--secured_port"; - String ARG_ZOOKEEPER_QUORUM = "--zk-quorum"; - String ARG_ZOOKEEPER_REGISTRY_PATH = "--zk-reg-path"; - String ARG_DEBUG = "--debug"; - String AGENT_MAIN_SCRIPT_ROOT = "./infra/agent/slider-agent/"; - String AGENT_JINJA2_ROOT = "./infra/agent/slider-agent/jinja2"; - String AGENT_MAIN_SCRIPT = "agent/main.py"; - - String APP_DEF = "application.def"; - String APP_DEF_ORIGINAL = "application.def.original"; - String ADDON_PREFIX = "application.addon."; - String ADDONS = "application.addons"; - String AGENT_VERSION = "agent.version"; - String AGENT_CONF = "agent.conf"; - String ADDON_FOR_ALL_COMPONENTS = "ALL"; - - String AGENT_INSTALL_DIR = "infra/agent"; - String APP_DEFINITION_DIR = "app/definition"; - String ADDON_DEFINITION_DIR = "addon/definition"; - String AGENT_CONFIG_FILE = "infra/conf/agent.ini"; - String AGENT_VERSION_FILE = "infra/version"; - - String PACKAGE_LIST = "package_list"; - String WAIT_HEARTBEAT = "wait.heartbeat"; - String PYTHON_EXE = "python"; - String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node"; - String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval"; - String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data"; - String AGENT_OUT_FILE = "slider-agent.out"; - String KEY_AGENT_TWO_WAY_SSL_ENABLED = "ssl.server.client.auth"; - String INFRA_RUN_SECURITY_DIR = "infra/run/security/"; - String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt"; - String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec"; - String TEST_RELAX_VERIFICATION = "test.relax.validation"; - - String DEFAULT_METAINFO_MAP_KEY = "DEFAULT_KEY"; -} - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java index 1d5d8a007c..47733272ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java @@ -17,37 +17,37 @@ */ package org.apache.slider.providers.docker; -import org.apache.hadoop.conf.Configuration; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.FileSystem; +import org.apache.slider.api.resource.Artifact; +import org.apache.slider.api.resource.ConfigFile; import org.apache.slider.common.SliderKeys; import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.providers.ProviderUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.slider.util.RestApiErrorMessages; -import java.util.Collections; -import java.util.List; +import java.io.IOException; public class DockerClientProvider extends AbstractClientProvider implements SliderKeys { - protected static final Logger log = - LoggerFactory.getLogger(DockerClientProvider.class); - private static final ProviderUtils providerUtils = new ProviderUtils(log); - protected static final String NAME = "docker"; - - public DockerClientProvider(Configuration conf) { - super(conf); + public DockerClientProvider() { + super(); } @Override - public String getName() { - return NAME; + public void validateArtifact(Artifact artifact, FileSystem fileSystem) { + if (artifact == null) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_INVALID); + } + if (StringUtils.isEmpty(artifact.getId())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); + } } @Override - public List getRoles() { - return Collections.emptyList(); + protected void validateConfigFile(ConfigFile configFile, FileSystem + fileSystem) throws IOException { } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java index 0e1d288424..7413ebbae5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java @@ -27,7 +27,4 @@ public interface DockerKeys { String DEFAULT_DOCKER_NETWORK = "bridge"; Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false; - - String OUT_FILE = "stdout.txt"; - String ERR_FILE = "stderr.txt"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java index 5d2592f9ab..6977e4179b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java @@ -17,27 +17,36 @@ */ package org.apache.slider.providers.docker; -import org.apache.hadoop.conf.Configuration; import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.ProviderService; import org.apache.slider.providers.SliderProviderFactory; public class DockerProviderFactory extends SliderProviderFactory { + private static final SliderProviderFactory FACTORY = new + DockerProviderFactory(); - public DockerProviderFactory() { + private DockerProviderFactory() { } - public DockerProviderFactory(Configuration conf) { - super(conf); + private static class Client { + static final AbstractClientProvider PROVIDER = new DockerClientProvider(); + } + + private static class Server { + static final ProviderService PROVIDER = new DockerProviderService(); } @Override public AbstractClientProvider createClientProvider() { - return new DockerClientProvider(getConf()); + return Client.PROVIDER; } @Override public ProviderService createServerProvider() { - return new DockerProviderService(); + return Server.PROVIDER; + } + + public static SliderProviderFactory getInstance() { + return FACTORY; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java index a48bf83693..f35d4d1a66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java @@ -17,159 +17,31 @@ */ package org.apache.slider.providers.docker; -import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Component; -import org.apache.slider.api.resource.ContainerState; -import org.apache.slider.common.SliderKeys; import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.launch.CommandLineBuilder; import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.core.registry.docstore.PublishedConfiguration; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.providers.ProviderService; -import org.apache.slider.providers.ProviderUtils; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher; -import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; +import org.apache.slider.providers.AbstractProviderService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Map; -import java.util.Map.Entry; -import static org.apache.slider.util.ServiceApiUtil.$; - -public class DockerProviderService extends AbstractService - implements ProviderService, DockerKeys, SliderKeys { +public class DockerProviderService extends AbstractProviderService + implements DockerKeys { protected static final Logger log = LoggerFactory.getLogger(DockerProviderService.class); - private static final ProviderUtils providerUtils = new ProviderUtils(log); - private static final String QUICK_LINKS = "quicklinks"; - protected StateAccessForProviders amState; - protected YarnRegistryViewForProviders yarnRegistry; - private ServiceTimelinePublisher serviceTimelinePublisher; protected DockerProviderService() { - super("DockerProviderService"); + super(DockerProviderService.class.getSimpleName()); } - @Override - public void setAMState(StateAccessForProviders stateAccessor) { - this.amState = stateAccessor; - } - - @Override - public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) { - this.yarnRegistry = yarnRegistry; - } - - - public void buildContainerLaunchContext(ContainerLauncher launcher, - Application application, Container container, ProviderRole providerRole, - SliderFileSystem fileSystem, RoleInstance roleInstance) - throws IOException, SliderException { - Component component = providerRole.component; + public void processArtifact(ContainerLauncher launcher, Component + component, SliderFileSystem fileSystem) throws IOException { launcher.setYarnDockerMode(true); launcher.setDockerImage(component.getArtifact().getId()); launcher.setDockerNetwork(component.getConfiguration() .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK)); launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer()); - - // Generate tokens (key-value pair) for config substitution. - // Get pre-defined tokens - Map globalTokens = amState.getGlobalSubstitutionTokens(); - Map tokensForSubstitution = providerUtils - .initCompTokensForSubstitute(roleInstance); - tokensForSubstitution.putAll(globalTokens); - // Set the environment variables in launcher - launcher.putEnv(SliderUtils - .buildEnvMap(component.getConfiguration(), tokensForSubstitution)); - launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$()); - launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); - if (System.getenv(HADOOP_USER_NAME) != null) { - launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME)); - } - launcher.setEnv("LANG", "en_US.UTF-8"); - launcher.setEnv("LC_ALL", "en_US.UTF-8"); - launcher.setEnv("LANGUAGE", "en_US.UTF-8"); - - for (Entry entry : launcher.getEnv().entrySet()) { - tokensForSubstitution.put($(entry.getKey()), entry.getValue()); - } - providerUtils.addComponentHostTokens(tokensForSubstitution, amState); - - // create config file on hdfs and add local resource - providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem, - component, tokensForSubstitution, roleInstance, amState); - - // substitute launch command - String launchCommand = ProviderUtils - .substituteStrWithTokens(component.getLaunchCommand(), - tokensForSubstitution); - CommandLineBuilder operation = new CommandLineBuilder(); - operation.add(launchCommand); - operation.addOutAndErrFiles(OUT_FILE, ERR_FILE); - launcher.addCommand(operation.build()); - - // publish exports - providerUtils - .substituteMapWithTokens(application.getQuicklinks(), tokensForSubstitution); - PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS, - application.getQuicklinks().entrySet()); - amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf); - if (serviceTimelinePublisher != null) { - serviceTimelinePublisher.serviceAttemptUpdated(application); - } - } - - public boolean processContainerStatus(ContainerId containerId, - ContainerStatus status) { - log.debug("Handling container status: {}", status); - if (SliderUtils.isEmpty(status.getIPs()) || - SliderUtils.isUnset(status.getHost())) { - return true; - } - RoleInstance instance = amState.getOwnedContainer(containerId); - if (instance == null) { - // container is completed? - return false; - } - - try { - providerUtils.updateServiceRecord(amState, yarnRegistry, - containerId.toString(), instance.role, status.getIPs(), status.getHost()); - } catch (IOException e) { - // could not write service record to ZK, log and retry - log.warn("Error updating container {} service record in registry, " + - "retrying", containerId, e); - return true; - } - // TODO publish ip and host - org.apache.slider.api.resource.Container container = - instance.providerRole.component.getContainer(containerId.toString()); - if (container != null) { - container.setIp(StringUtils.join(",", status.getIPs())); - container.setHostname(status.getHost()); - container.setState(ContainerState.READY); - } else { - log.warn(containerId + " not found in Application!"); - } - return false; - } - - @Override - public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) { - this.serviceTimelinePublisher = publisher; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java new file mode 100644 index 0000000000..3e020cf993 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.tarball; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.slider.api.resource.Artifact; +import org.apache.slider.api.resource.ConfigFile; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.providers.AbstractClientProvider; +import org.apache.slider.util.RestApiErrorMessages; + +import java.io.IOException; +import java.nio.file.Paths; + +public class TarballClientProvider extends AbstractClientProvider + implements SliderKeys { + + public TarballClientProvider() { + } + + @Override + public void validateArtifact(Artifact artifact, FileSystem fs) + throws IOException { + if (artifact == null) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_INVALID); + } + if (StringUtils.isEmpty(artifact.getId())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); + } + Path p = new Path(artifact.getId()); + if (!fs.exists(p)) { + throw new IllegalArgumentException( "Artifact tarball does not exist " + + artifact.getId()); + } + } + + @Override + protected void validateConfigFile(ConfigFile configFile, FileSystem + fileSystem) throws IOException { + // validate dest_file is not absolute + if (Paths.get(configFile.getDestFile()).isAbsolute()) { + throw new IllegalArgumentException( + "Dest_file must not be absolute path: " + configFile.getDestFile()); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java new file mode 100644 index 0000000000..d9b745023e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.tarball; + +import org.apache.slider.providers.AbstractClientProvider; +import org.apache.slider.providers.ProviderService; +import org.apache.slider.providers.SliderProviderFactory; + +public class TarballProviderFactory extends SliderProviderFactory { + private static final SliderProviderFactory FACTORY = new + TarballProviderFactory(); + + private TarballProviderFactory() { + } + + private static class Client { + static final AbstractClientProvider PROVIDER = new TarballClientProvider(); + } + + private static class Server { + static final ProviderService PROVIDER = new TarballProviderService(); + } + + @Override + public AbstractClientProvider createClientProvider() { + return Client.PROVIDER; + } + + @Override + public ProviderService createServerProvider() { + return Server.PROVIDER; + } + + public static SliderProviderFactory getInstance() { + return FACTORY; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java new file mode 100644 index 0000000000..65a55f0623 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.tarball; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.slider.api.resource.Component; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.providers.AbstractProviderService; + +import java.io.IOException; + +public class TarballProviderService extends AbstractProviderService { + + protected TarballProviderService() { + super(TarballProviderService.class.getSimpleName()); + } + + @Override + public void processArtifact(ContainerLauncher launcher, Component + component, SliderFileSystem fileSystem) throws IOException { + Path artifact = new Path(component.getArtifact().getId()); + if (!fileSystem.isFile(artifact)) { + throw new IOException("Package doesn't exist as a resource: " + + artifact.toString()); + } + log.info("Adding resource {}", artifact.toString()); + LocalResourceType type = LocalResourceType.ARCHIVE; + LocalResource packageResource = fileSystem.createAmResource( + artifact, type); + launcher.addLocalResource(APP_INSTALL_DIR, packageResource); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java index 7c096c72aa..aa84940b9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java @@ -27,7 +27,7 @@ import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderService; -import org.apache.slider.providers.agent.AgentKeys; +import org.apache.slider.providers.SliderProviderFactory; import org.apache.slider.server.appmaster.actions.ActionStartContainer; import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.state.ContainerAssignment; @@ -43,6 +43,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import static org.apache.slider.common.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY; + /** * A service for launching containers */ @@ -59,11 +61,6 @@ public class RoleLaunchService */ private final QueueAccess actionQueue; - /** - * Provider building up the command - */ - private final ProviderService provider; - /** * Filesystem to use for the launch */ @@ -75,16 +72,14 @@ public class RoleLaunchService /** * Construct an instance of the launcher * @param queueAccess - * @param provider the provider * @param fs filesystem * @param envVars environment variables */ - public RoleLaunchService(QueueAccess queueAccess, ProviderService provider, - SliderFileSystem fs, Map envVars) { + public RoleLaunchService(QueueAccess queueAccess, SliderFileSystem fs, + Map envVars) { super(ROLE_LAUNCH_SERVICE); this.actionQueue = queueAccess; this.fs = fs; - this.provider = provider; this.envVars = envVars; } @@ -167,11 +162,13 @@ public void run() { instance.roleId = role.id; instance.environment = envDescription; + ProviderService provider = SliderProviderFactory.getProviderService( + role.component.getArtifact()); provider.buildContainerLaunchContext(containerLauncher, application, container, role, fs, instance); long delay = role.component.getConfiguration() - .getPropertyLong(AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0); + .getPropertyLong(KEY_CONTAINER_LAUNCH_DELAY, 0); long maxDelay = getConfig() .getLong(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 02c919864c..84dde081b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.client.api.TimelineV2Client; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; @@ -342,7 +341,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * ProviderService of this cluster */ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized") - private ProviderService providerService; + private List providers = new ArrayList<>(); /** * The YARN registry service @@ -523,8 +522,6 @@ private void startQueueProcessing() { public Configuration bindArgs(Configuration config, String... args) throws Exception { // let the superclass process it Configuration superConf = super.bindArgs(config, args); - // add the slider XML config - ConfigHelper.injectSliderXMLResource(); //yarn-ify YarnConfiguration yarnConfiguration = new YarnConfiguration( @@ -603,12 +600,15 @@ private int createAndRunCluster(String appName) throws Throwable { // obtain security state // set the global security flag for the instance definition - //get our provider - SliderProviderFactory factory = - SliderProviderFactory.createSliderProviderFactory("docker"); - providerService = factory.createServerProvider(); - // init the provider BUT DO NOT START IT YET - initAndAddService(providerService); + // initialize our providers + for (Component component : application.getComponents()) { + SliderProviderFactory factory = SliderProviderFactory + .createSliderProviderFactory(component.getArtifact()); + ProviderService providerService = factory.createServerProvider(); + // init the provider BUT DO NOT START IT YET + initAndAddService(providerService); + providers.add(providerService); + } InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf); log.info("RM is at {}", rmSchedulerAddress); @@ -667,7 +667,9 @@ private int createAndRunCluster(String appName) throws Throwable { serviceTimelinePublisher.init(getConfig()); serviceTimelinePublisher.start(); - providerService.setServiceTimelinePublisher(serviceTimelinePublisher); + for (ProviderService providerService : providers) { + providerService.setServiceTimelinePublisher(serviceTimelinePublisher); + } appState.setServiceTimelinePublisher(serviceTimelinePublisher); log.info("ServiceTimelinePublisher started."); } @@ -707,7 +709,7 @@ private int createAndRunCluster(String appName) throws Throwable { WebAppApiImpl webAppApi = new WebAppApiImpl( stateForProviders, - providerService, registryOperations, + registryOperations, metricsAndMonitoring, actionQueues); initAMFilterOptions(serviceConf); @@ -843,13 +845,14 @@ private int createAndRunCluster(String appName) throws Throwable { //launcher service launchService = new RoleLaunchService(actionQueues, - providerService, fs, envVars); deployChildService(launchService); //Give the provider access to the state, and AM - providerService.setAMState(stateForProviders); + for (ProviderService providerService : providers) { + providerService.setAMState(stateForProviders); + } // chaos monkey maybeStartMonkey(); @@ -1119,7 +1122,9 @@ public void registerServiceInstance(String instanceName, SliderKeys.APP_TYPE, instanceName, appAttemptID); - providerService.bindToYarnRegistry(yarnRegistryOperations); + for (ProviderService providerService : providers) { + providerService.bindToYarnRegistry(yarnRegistryOperations); + } // Yarn registry ServiceRecord serviceRecord = new ServiceRecord(); @@ -1859,7 +1864,9 @@ public void updateBlacklist(List blacklistAdditions, protected synchronized void launchProviderService() throws IOException, SliderException { // didn't start, so don't register - providerService.start(); + for (ProviderService providerService : providers) { + providerService.start(); + } // and send the started event ourselves eventCallbackEvent(null); } @@ -1959,19 +1966,23 @@ public void onContainerStatusReceived(ContainerId containerId, ContainerStatus containerStatus) { LOG_YARN.debug("Container Status: id={}, status={}", containerId, containerStatus); + RoleInstance cinfo = appState.getOwnedContainer(containerId); + if (cinfo == null) { + LOG_YARN.error("Owned container not found for {}", containerId); + return; + } + ProviderService providerService = SliderProviderFactory + .getProviderService(cinfo.providerRole.component.getArtifact()); if (providerService.processContainerStatus(containerId, containerStatus)) { try { Thread.sleep(1000); } catch (InterruptedException e) { } - RoleInstance cinfo = appState.getOwnedContainer(containerId); - if (cinfo != null) { - LOG_YARN.info("Re-requesting status for role {}, {}", - cinfo.role, containerId); - //trigger another async container status - nmClientAsync.getContainerStatusAsync(containerId, - cinfo.container.getNodeId()); - } + LOG_YARN.info("Re-requesting status for role {}, {}", + cinfo.role, containerId); + //trigger another async container status + nmClientAsync.getContainerStatusAsync(containerId, + cinfo.container.getNodeId()); } else if (timelineServiceEnabled) { RoleInstance instance = appState.getOwnedContainer(containerId); if (instance != null) { @@ -1997,11 +2008,6 @@ public void onStopContainerError(ContainerId containerId, Throwable t) { LOG_YARN.warn("Failed to stop Container {}", containerId); } - - public ProviderService getProviderService() { - return providerService; - } - /** * Queue an action for immediate execution in the executor thread * @param action action to execute diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java index 094726d1c4..02f3f0c3db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java @@ -17,7 +17,6 @@ package org.apache.slider.server.appmaster.web; import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.slider.providers.ProviderService; import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.state.AppState; @@ -33,11 +32,6 @@ public interface WebAppApi { */ StateAccessForProviders getAppState(); - /** - * The {@link ProviderService} for the current cluster - */ - ProviderService getProviderService(); - /** * Registry operations accessor * @return registry access diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java index fd9381c5c9..f88f501faa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java @@ -17,7 +17,6 @@ package org.apache.slider.server.appmaster.web; import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.slider.providers.ProviderService; import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.state.StateAccessForProviders; @@ -33,21 +32,18 @@ public class WebAppApiImpl implements WebAppApi { private static final Logger log = LoggerFactory.getLogger(WebAppApiImpl.class); protected final StateAccessForProviders appState; - protected final ProviderService provider; private final RegistryOperations registryOperations; private final MetricsAndMonitoring metricsAndMonitoring; private final QueueAccess queues; public WebAppApiImpl(StateAccessForProviders appState, - ProviderService provider, RegistryOperations registryOperations, + RegistryOperations registryOperations, MetricsAndMonitoring metricsAndMonitoring, QueueAccess queues) { checkNotNull(appState); - checkNotNull(provider); this.queues = queues; this.registryOperations = registryOperations; this.appState = appState; - this.provider = provider; this.metricsAndMonitoring = metricsAndMonitoring; } @@ -56,11 +52,6 @@ public StateAccessForProviders getAppState() { return appState; } - @Override - public ProviderService getProviderService() { - return provider; - } - @Override public RegistryOperations getRegistryOperations() { return registryOperations; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java index 82d7c8f57a..5f44bda1e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java @@ -19,7 +19,6 @@ package org.apache.slider.server.appmaster.web.view; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; -import org.apache.slider.providers.ProviderService; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.WebAppApi; import org.apache.slider.server.appmaster.web.rest.RestPaths; @@ -33,12 +32,10 @@ public abstract class SliderHamletBlock extends HtmlBlock { protected final StateAccessForProviders appState; - protected final ProviderService providerService; protected final RestPaths restPaths = new RestPaths(); public SliderHamletBlock(WebAppApi slider) { this.appState = slider.getAppState(); - this.providerService = slider.getProviderService(); } protected String rootPath(String absolutePath) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml deleted file mode 100644 index 96bfe0fc08..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - slider.config.loaded - true - - - slider.provider.docker - org.apache.slider.providers.docker.DockerProviderFactory - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java index 11abdfedbf..e58d981f96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java @@ -18,11 +18,16 @@ package org.apache.slider.providers; -import org.apache.slider.providers.docker.DockerKeys; +import org.apache.slider.api.resource.Artifact; +import org.apache.slider.api.resource.Artifact.TypeEnum; +import org.apache.slider.providers.docker.DockerClientProvider; import org.apache.slider.providers.docker.DockerProviderFactory; +import org.apache.slider.providers.docker.DockerProviderService; +import org.apache.slider.providers.tarball.TarballClientProvider; +import org.apache.slider.providers.tarball.TarballProviderFactory; +import org.apache.slider.providers.tarball.TarballProviderService; import org.junit.Test; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -30,25 +35,38 @@ */ public class TestProviderFactory { @Test - public void testLoadAgentProvider() throws Throwable { + public void testDockerFactory() throws Throwable { SliderProviderFactory factory = SliderProviderFactory - .createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER); + .createSliderProviderFactory(new Artifact().type(TypeEnum.DOCKER)); assertTrue(factory instanceof DockerProviderFactory); + assertTrue(factory.createClientProvider() instanceof DockerClientProvider); + assertTrue(factory.createServerProvider() instanceof DockerProviderService); + assertTrue(SliderProviderFactory.getProviderService(new Artifact() + .type(TypeEnum.DOCKER)) instanceof DockerProviderService); } @Test - public void testCreateClientProvider() throws Throwable { + public void testTarballFactory() throws Throwable { SliderProviderFactory factory = SliderProviderFactory - .createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER); - assertNotNull(factory.createClientProvider()); + .createSliderProviderFactory(new Artifact().type(TypeEnum.TARBALL)); + assertTrue(factory instanceof TarballProviderFactory); + assertTrue(factory.createClientProvider() instanceof TarballClientProvider); + assertTrue(factory.createServerProvider() instanceof + TarballProviderService); + assertTrue(SliderProviderFactory.getProviderService(new Artifact() + .type(TypeEnum.TARBALL)) instanceof TarballProviderService); } @Test - public void testCreateProviderByClassname() throws Throwable { + public void testDefaultFactory() throws Throwable { SliderProviderFactory factory = SliderProviderFactory - .createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER); - assertNotNull(factory.createServerProvider()); - assertTrue(factory instanceof DockerProviderFactory); + .createSliderProviderFactory(null); + assertTrue(factory instanceof DefaultProviderFactory); + assertTrue(factory.createClientProvider() instanceof DefaultClientProvider); + assertTrue(factory.createServerProvider() instanceof + DefaultProviderService); + assertTrue(SliderProviderFactory.getProviderService(null) instanceof + DefaultProviderService); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java index e4e344e02a..bc6cfd0b3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java @@ -20,9 +20,7 @@ import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; -import org.apache.slider.providers.ProviderService; import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockProviderService; import org.apache.slider.server.appmaster.state.ProviderAppState; import org.apache.slider.server.appmaster.web.WebAppApi; import org.apache.slider.server.appmaster.web.WebAppApiImpl; @@ -45,11 +43,9 @@ public void setup() throws Exception { ProviderAppState providerAppState = new ProviderAppState( "undefined", appState); - ProviderService providerService = new MockProviderService(); WebAppApiImpl inst = new WebAppApiImpl( providerAppState, - providerService, null, null, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java index 92f8559be8..33385dbe2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java @@ -27,12 +27,10 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR; import org.apache.hadoop.yarn.webapp.hamlet.HamletImpl.EImp; import org.apache.slider.api.ClusterNode; -import org.apache.slider.providers.ProviderService; import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; import org.apache.slider.server.appmaster.model.mock.MockContainer; import org.apache.slider.server.appmaster.model.mock.MockContainerId; import org.apache.slider.server.appmaster.model.mock.MockNodeId; -import org.apache.slider.server.appmaster.model.mock.MockProviderService; import org.apache.slider.server.appmaster.model.mock.MockResource; import org.apache.slider.server.appmaster.state.ProviderAppState; import org.apache.slider.server.appmaster.state.RoleInstance; @@ -65,14 +63,12 @@ public class TestContainerStatsBlock extends BaseMockAppStateTest { @Before public void setup() throws Exception { super.setup(); - ProviderService providerService = new MockProviderService(); ProviderAppState providerAppState = new ProviderAppState( "undefined", appState); WebAppApiImpl inst = new WebAppApiImpl( providerAppState, - providerService, null, METRICS, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java index 96eb3d7ef0..b2d06374c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java @@ -22,12 +22,10 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; -import org.apache.slider.providers.ProviderService; import org.apache.slider.server.appmaster.model.appstate.BaseMockAppStateAATest; import org.apache.slider.server.appmaster.model.mock.MockContainer; import org.apache.slider.server.appmaster.model.mock.MockContainerId; import org.apache.slider.server.appmaster.model.mock.MockNodeId; -import org.apache.slider.server.appmaster.model.mock.MockProviderService; import org.apache.slider.server.appmaster.model.mock.MockResource; import org.apache.slider.server.appmaster.state.ContainerOutcome; import org.apache.slider.server.appmaster.state.OutstandingRequest; @@ -58,14 +56,12 @@ public class TestIndexBlock extends BaseMockAppStateAATest { public void setup() throws Exception { super.setup(); assertNotNull(appState); - ProviderService providerService = new MockProviderService(); ProviderAppState providerAppState = new ProviderAppState( "undefined", appState); WebAppApiImpl inst = new WebAppApiImpl( providerAppState, - providerService, null, METRICS, null);