big-data/hadoop
big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

YARN-5461. Initial code ported from slider-core module. (jianhe)

Browse Source
This commit is contained in:
Jian He 2016-08-03 00:18:01 +08:00
parent dcd99c4b9a
commit f47df51791
461 changed files with 109486 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

397
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml Normal file
View File

@ -0,0 +1,397 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-slider-core</artifactId>
<packaging>jar</packaging>
<name>Apache Hadoop YARN Slider Core</name>
<build>
<!-- resources are filtered for dynamic updates. This gets build info in-->
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<environmentVariables>
<JAVA_HOME>${java.home}</JAVA_HOME>
</environmentVariables>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.avro</groupId>
<artifactId>avro-maven-plugin</artifactId>
<version>${avro.version}</version>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>schema</goal>
</goals>
<configuration>
<sourceDirectory>${project.basedir}/src/main/avro/
</sourceDirectory>
<outputDirectory>${project.build.directory}/generated-sources/java
</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
<version>1.30</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-jaxrs</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-xc</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-registry</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</dependency>
<dependency>
<groupId>commons-digester</groupId>
<artifactId>commons-digester</artifactId>
<version>1.8</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-core</artifactId>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-servlets</artifactId>
<version>3.0.1</version>
</dependency>
<!-- ======================================================== -->
<!-- service registry -->
<!-- ======================================================== -->
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
<!-- ======================================================== -->
<!-- Jersey and webapp support -->
<!-- ======================================================== -->
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
</dependency>
<dependency>
<groupId>com.sun.jersey.contribs</groupId>
<artifactId>jersey-guice</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
</dependency>
<dependency>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
<version>6.1.26</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.16</version>
<scope>compile</scope>
</dependency>
</dependencies>
<profiles>
<profile>
<id>compile-protobuf</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<executions>
<execution>
<id>compile-protoc</id>
<phase>generate-sources</phase>
<goals>
<goal>protoc</goal>
</goals>
<configuration>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>protoc</protocCommand>
<imports>
<param>${basedir}/src/main/proto</param>
</imports>
<source>
<directory>${basedir}/src/main/proto</directory>
<includes>
<include>SliderClusterMessages.proto</include>
<include>SliderClusterProtocol.proto</include>
</includes>
</source>
<output>${basedir}/src/main/java</output>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>rat</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<executions>
<execution>
<id>check-licenses</id>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
<configuration>
<excludes>
<exclude>**/*.json</exclude>
<exclude>src/test/python/agent.ini</exclude>
<exclude>src/test/python/version</exclude>
<exclude>**/THIRD-PARTY.properties</exclude>
<exclude>src/main/resources/webapps/slideram/.keep</exclude>
<exclude>src/main/resources/webapps/slideragent/.keep</exclude>
<exclude>src/main/resources/webapps/static/yarn.dt.plugins.js</exclude>
<!-- jQuery DataTables files (BSD license) -->
<exclude>src/main/resources/webapps/static/dt-1.9.4/**</exclude>
<!-- jQuery (MIT license) -->
<exclude>src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js</exclude>
<!-- jQuery UI (MIT license) -->
<exclude>src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js</exclude>
<exclude>src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css</exclude>
<!-- jQuery jsTree (MIT license) -->
<exclude>src/main/resources/webapps/static/jt/jquery.jstree.js</exclude>
<!-- protobuf generated classes -->
<exclude>src/main/java/org/apache/slider/api/proto/Messages.java</exclude>
<exclude>src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java</exclude>
<exclude>src/test/app_packages/test_am_config/resources/test.template</exclude>
<exclude>src/test/app_packages/test_am_config/test_archive/testfile</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml Normal file
View File

@ -0,0 +1,47 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>distribution</id>
<formats>
<format>zip</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>${project.build.directory}</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<scope>runtime</scope>
<outputDirectory>/lib</outputDirectory>
<!-- dont copy JAR into /lib-->
<useProjectArtifact>false</useProjectArtifact>
<!-- <includeBaseDirectory>false</includeBaseDirectory>-->
<unpack>false</unpack>
</dependencySet>
</dependencySets>
</assembly>

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties Normal file
View File

@ -0,0 +1,33 @@
# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
#-------------------------------------------------------------------------------
# Already used licenses in project :
# - Apache License
# - BSD
# - CDDL + GPLv2 with classpath exception
# - CDDL 1.1
# - CDDL License
# - CDDL+GPL
# - Common Public License Version 1.0
# - Eclipse Public License - Version 1.0
# - GNU Lesser General Public License (LGPL), Version 2.1
# - GNU Lesser General Public License, Version 2.1
# - GPL2 w/ CPE
# - MIT License
# - MPL 1.1
# - New BSD License
# - Public Domain
# - Revised BSD
# - The Apache Software License, Version 2.0
# - The BSD 3-Clause License
# - The BSD License
# - The MIT License
#-------------------------------------------------------------------------------
# Please fill the missing licenses for dependencies :
#
#
#Thu Oct 15 16:45:02 EDT 2015
commons-beanutils--commons-beanutils--1.7.0=The Apache Software License, Version 2.0
javax.servlet--servlet-api--2.5=CDDL License
javax.servlet.jsp--jsp-api--2.1=CDDL License
org.apache.zookeeper--zookeeper--3.4.6=The Apache Software License, Version 2.0
org.codehaus.jettison--jettison--1.1=The Apache Software License, Version 2.0

114
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc Normal file
View File

@ -0,0 +1,114 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
[
{
"type": "record",
"name": "NodeEntryRecord",
"namespace": "org.apache.slider.server.avro",
"fields": [
{
"name": "host",
"type": "string"
},
{
"name": "role",
"type": "int"
},
{
"name": "active",
"type": "boolean"
},
{
"name": "last_used",
"type": "long"
}
]
},
{
"type": "record",
"name": "RoleHistoryHeader",
"namespace": "org.apache.slider.server.avro",
"fields": [
{
"name": "version",
"type": "int"
},
{
"name": "saved",
"type": "long"
},
{
"name": "savedx",
"type": "string"
},
{
"name": "savedate",
"type": "string",
"default": ""
},
{
"name": "roles",
"type": "int"
}
]
},
{
"type": "record",
"name": "RoleHistoryMapping",
"namespace": "org.apache.slider.server.avro",
"fields": [
{
"name": "rolemap",
"type": {
"type": "map",
"values": "int"
}
}
]
},
{
"type": "record",
"name": "RoleHistoryFooter",
"namespace": "org.apache.slider.server.avro",
"fields": [
{
"name": "count",
"type": "long"
}
]
},
{
"type": "record",
"name": "RoleHistoryRecord",
"namespace": "org.apache.slider.server.avro",
"fields": [
{
"name": "entry",
"type": [
"org.apache.slider.server.avro.NodeEntryRecord",
"org.apache.slider.server.avro.RoleHistoryHeader",
"org.apache.slider.server.avro.RoleHistoryFooter",
"org.apache.slider.server.avro.RoleHistoryMapping"
]
}
]
}
]

676
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java Normal file
View File

@ -0,0 +1,676 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.security;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.Shell;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.crypto.Cipher;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.reflect.InvocationTargetException;
import java.net.InetAddress;
import java.security.NoSuchAlgorithmException;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.regex.Pattern;
import static org.apache.hadoop.security.UserGroupInformation.*;
import static org.apache.hadoop.security.authentication.util.KerberosUtil.*;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
/**
* Kerberos diagnostics
* At some point this may move to hadoop core, so please keep use of slider
* methods and classes to ~0.
*
* This operation expands some of the diagnostic output of the security code,
* but not all. For completeness
*
* Set the environment variable {@code HADOOP_JAAS_DEBUG=true}
* Set the log level for {@code org.apache.hadoop.security=DEBUG}
*/
public class KerberosDiags implements Closeable {
private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class);
public static final String KRB5_CCNAME = "KRB5CCNAME";
public static final String JAVA_SECURITY_KRB5_CONF
= "java.security.krb5.conf";
public static final String JAVA_SECURITY_KRB5_REALM
= "java.security.krb5.realm";
public static final String SUN_SECURITY_KRB5_DEBUG
= "sun.security.krb5.debug";
public static final String SUN_SECURITY_SPNEGO_DEBUG
= "sun.security.spnego.debug";
public static final String SUN_SECURITY_JAAS_FILE
= "java.security.auth.login.config";
public static final String KERBEROS_KINIT_COMMAND
= "hadoop.kerberos.kinit.command";
public static final String HADOOP_AUTHENTICATION_IS_DISABLED
= "Hadoop authentication is disabled";
public static final String UNSET = "(unset)";
public static final String NO_DEFAULT_REALM = "Cannot locate default realm";
private final Configuration conf;
private final List<String> services;
private final PrintWriter out;
private final File keytab;
private final String principal;
private final long minKeyLength;
private final boolean securityRequired;
public static final String CAT_JVM = "JVM";
public static final String CAT_JAAS = "JAAS";
public static final String CAT_CONFIG = "CONFIG";
public static final String CAT_LOGIN = "LOGIN";
public static final String CAT_KERBEROS = "KERBEROS";
public static final String CAT_SASL = "SASL";
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
public KerberosDiags(Configuration conf,
PrintWriter out,
List<String> services,
File keytab,
String principal,
long minKeyLength,
boolean securityRequired) {
this.conf = conf;
this.services = services;
this.keytab = keytab;
this.principal = principal;
this.out = out;
this.minKeyLength = minKeyLength;
this.securityRequired = securityRequired;
}
@Override
public void close() throws IOException {
flush();
}
/**
* Execute diagnostics.
* <p>
* Things it would be nice if UGI made accessible
* <ol>
* <li>A way to enable JAAS debug programatically</li>
* <li>Access to the TGT</li>
* </ol>
* @return true if security was enabled and all probes were successful
* @throws KerberosDiagsFailure explicitly raised failure
* @throws Exception other security problems
*/
@SuppressWarnings("deprecation")
public boolean execute() throws Exception {
title("Kerberos Diagnostics scan at %s",
new Date(System.currentTimeMillis()));
// check that the machine has a name
println("Hostname: %s",
InetAddress.getLocalHost().getCanonicalHostName());
// Fail fast on a JVM without JCE installed.
validateKeyLength();
// look at realm
println("JVM Kerberos Login Module = %s", getKrb5LoginModuleName());
printDefaultRealm();
title("System Properties");
for (String prop : new String[]{
JAVA_SECURITY_KRB5_CONF,
JAVA_SECURITY_KRB5_REALM,
SUN_SECURITY_KRB5_DEBUG,
SUN_SECURITY_SPNEGO_DEBUG,
SUN_SECURITY_JAAS_FILE
}) {
printSysprop(prop);
}
title("Environment Variables");
for (String env : new String[]{
"HADOOP_JAAS_DEBUG",
KRB5_CCNAME,
HADOOP_USER_NAME,
HADOOP_PROXY_USER,
HADOOP_TOKEN_FILE_LOCATION,
}) {
printEnv(env);
}
for (String prop : new String[]{
KERBEROS_KINIT_COMMAND,
HADOOP_SECURITY_AUTHENTICATION,
HADOOP_SECURITY_AUTHORIZATION,
"hadoop.kerberos.min.seconds.before.relogin", // not in 2.6
"hadoop.security.dns.interface", // not in 2.6
"hadoop.security.dns.nameserver", // not in 2.6
HADOOP_RPC_PROTECTION,
HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX,
HADOOP_SECURITY_GROUP_MAPPING,
"hadoop.security.impersonation.provider.class", // not in 2.6
"dfs.data.transfer.protection" // HDFS
}) {
printConfOpt(prop);
}
// check that authentication is enabled
if (SecurityUtil.getAuthenticationMethod(conf)
.equals(AuthenticationMethod.SIMPLE)) {
println(HADOOP_AUTHENTICATION_IS_DISABLED);
failif(securityRequired, CAT_CONFIG, HADOOP_AUTHENTICATION_IS_DISABLED);
// no security, skip rest of test
return false;
}
validateKrb5File();
validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS);
validateSasl("dfs.data.transfer.saslproperties.resolver.class");
validateKinitExecutable();
validateJAAS();
// now the big test: login, then try again
boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG);
boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG);
try {
title("Logging in");
if (keytab != null) {
dumpKeytab(keytab);
loginFromKeytab();
} else {
UserGroupInformation loginUser = getLoginUser();
dumpUGI("Log in user", loginUser);
validateUGI("Login user", loginUser);
println("Ticket based login: %b", isLoginTicketBased());
println("Keytab based login: %b", isLoginKeytabBased());
}
return true;
} finally {
// restore original system properties
System.setProperty(SUN_SECURITY_KRB5_DEBUG,
Boolean.toString(krb5Debug));
System.setProperty(SUN_SECURITY_SPNEGO_DEBUG,
Boolean.toString(spnegoDebug));
}
}
/**
* Fail fast on a JVM without JCE installed.
*
* This is a recurrent problem
* (that is: it keeps creeping back with JVM updates);
* a fast failure is the best tactic
* @throws NoSuchAlgorithmException
*/
protected void validateKeyLength() throws NoSuchAlgorithmException {
int aesLen = Cipher.getMaxAllowedKeyLength("AES");
println("Maximum AES encryption key length %d bits", aesLen);
failif (aesLen < minKeyLength,
CAT_JVM,
"Java Cryptography Extensions are not installed on this JVM."
+" Maximum supported key length %s - minimum required %d",
aesLen, minKeyLength);
}
/**
* Get the default realm.
* <p>
* Not having a default realm may be harmless, so is noted at info.
* All other invocation failures are downgraded to warn, as
* follow-on actions may still work.
* failure to invoke the method via introspection is rejected,
* as it's a sign of JVM compatibility issues that may have other
* consequences
*/
protected void printDefaultRealm() {
try {
println("Default Realm = %s",
getDefaultRealm());
} catch (ClassNotFoundException
| IllegalAccessException
| NoSuchMethodException e) {
throw new KerberosDiagsFailure(CAT_JVM, e,
"Failed to invoke krb5.Config.getDefaultRealm: %s", e);
} catch (InvocationTargetException e) {
Throwable cause = e.getCause() != null ? e.getCause() : e;
if (cause.toString().contains(NO_DEFAULT_REALM)) {
// exception raised if there is no default realm. This is not
// always a problem, so downgrade to a message.
println("Host has no default realm");
LOG.debug(cause.toString(), cause);
} else {
println("Kerberos.getDefaultRealm() failed: %s\n%s",
cause,
org.apache.hadoop.util.StringUtils.stringifyException(cause));
}
}
}
/**
* Locate the krb5.conf file and dump it.
* No-op on windows.
* @throws IOException
*/
private void validateKrb5File() throws IOException {
if (!Shell.WINDOWS) {
title("Locating Kerberos configuration file");
String krbPath = "/etc/krb5.conf";
String jvmKrbPath = System.getProperty(JAVA_SECURITY_KRB5_CONF);
if (jvmKrbPath != null) {
println("Setting kerberos path from sysprop %s: %s",
JAVA_SECURITY_KRB5_CONF, jvmKrbPath);
krbPath = jvmKrbPath;
}
String krb5name = System.getenv(KRB5_CCNAME);
if (krb5name != null) {
println("Setting kerberos path from environment variable %s: %s",
KRB5_CCNAME, krb5name);
krbPath = krb5name;
if (jvmKrbPath != null) {
println("Warning - both %s and %s were set - %s takes priority",
JAVA_SECURITY_KRB5_CONF, KRB5_CCNAME, KRB5_CCNAME);
}
}
File krbFile = new File(krbPath);
println("Kerberos configuration file = %s", krbFile);
failif(!krbFile.exists(),
CAT_KERBEROS,
"Kerberos configuration file %s not found", krbFile);
dump(krbFile);
}
}
/**
* Dump a keytab: list all principals.
* @param keytabFile the keytab file
* @throws IOException IO problems
*/
public void dumpKeytab(File keytabFile) throws IOException {
title("Examining keytab %s", keytabFile);
File kt = keytabFile.getCanonicalFile();
failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt);
failif(!kt.isFile(), CAT_CONFIG, "Keytab is not a valid file: %s", kt);
String[] names = getPrincipalNames(keytabFile.getCanonicalPath(),
Pattern.compile(".*"));
println("keytab entry count: %d", names.length);
for (String name : names) {
println(" %s", name);
}
println("-----");
}
/**
* Log in from a keytab, dump the UGI, validate it, then try and log in again.
* That second-time login catches JVM/Hadoop compatibility problems.
* @throws IOException
*/
private void loginFromKeytab() throws IOException {
UserGroupInformation ugi;
String identity;
if (keytab != null) {
File kt = keytab.getCanonicalFile();
println("Using keytab %s principal %s", kt, principal);
identity = principal;
failif(StringUtils.isEmpty(principal), CAT_KERBEROS,
"No principal defined");
ugi = loginUserFromKeytabAndReturnUGI(principal, kt.getPath());
dumpUGI(identity, ugi);
validateUGI(principal, ugi);
title("Attempting to log in from keytab again");
// package scoped -hence the reason why this class must be in the
// hadoop.security package
setShouldRenewImmediatelyForTests(true);
// attempt a new login
ugi.reloginFromKeytab();
} else {
println("No keytab: logging is as current user");
}
}
/**
* Dump a UGI.
* @param title title of this section
* @param ugi UGI to dump
* @throws IOException
*/
private void dumpUGI(String title, UserGroupInformation ugi)
throws IOException {
title(title);
println("UGI instance = %s", ugi);
println("Has kerberos credentials: %b", ugi.hasKerberosCredentials());
println("Authentication method: %s", ugi.getAuthenticationMethod());
println("Real Authentication method: %s",
ugi.getRealAuthenticationMethod());
title("Group names");
for (String name : ugi.getGroupNames()) {
println(name);
}
title("Credentials");
Credentials credentials = ugi.getCredentials();
List<Text> secretKeys = credentials.getAllSecretKeys();
title("Secret keys");
if (!secretKeys.isEmpty()) {
for (Text secret: secretKeys) {
println("%s", secret);
}
} else {
println("(none)");
}
dumpTokens(ugi);
}
/**
* Validate the UGI: verify it is kerberized.
* @param messagePrefix message in exceptions
* @param user user to validate
*/
private void validateUGI(String messagePrefix, UserGroupInformation user) {
failif(!user.hasKerberosCredentials(),
CAT_LOGIN, "%s: No kerberos credentials for %s", messagePrefix, user);
failif(user.getAuthenticationMethod() == null,
CAT_LOGIN, "%s: Null AuthenticationMethod for %s", messagePrefix, user);
}
/**
* A cursory look at the {@code kinit} executable.
* If it is an absolute path: it must exist with a size > 0.
* If it is just a command, it has to be on the path. There's no check
* for that -but the PATH is printed out.
*/
private void validateKinitExecutable() {
String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, "");
if (!kinit.isEmpty()) {
File kinitPath = new File(kinit);
println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath);
if (kinitPath.isAbsolute()) {
failif(!kinitPath.exists(), CAT_KERBEROS,
"%s executable does not exist: %s",
KERBEROS_KINIT_COMMAND, kinitPath);
failif(!kinitPath.isFile(), CAT_KERBEROS,
"%s path does not refer to a file: %s",
KERBEROS_KINIT_COMMAND, kinitPath);
failif(kinitPath.length() == 0, CAT_KERBEROS,
"%s file is empty: %s",
KERBEROS_KINIT_COMMAND, kinitPath);
} else {
println("Executable %s is relative -must be on the PATH", kinit);
printEnv("PATH");
}
}
}
/**
* Try to load the SASL resolver.
* @param saslPropsResolverKey key for the SASL resolver
*/
private void validateSasl(String saslPropsResolverKey) {
title("Resolving SASL property %s", saslPropsResolverKey);
String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey);
try {
Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass(
saslPropsResolverKey,
SaslPropertiesResolver.class, SaslPropertiesResolver.class);
println("Resolver is %s", resolverClass);
} catch (RuntimeException e) {
throw new KerberosDiagsFailure(CAT_SASL, e,
"Failed to load %s class %s",
saslPropsResolverKey, saslPropsResolver);
}
}
/**
* Validate any JAAS entry referenced in the {@link #SUN_SECURITY_JAAS_FILE}
* property.
*/
private void validateJAAS() {
String jaasFilename = System.getProperty(SUN_SECURITY_JAAS_FILE);
if (jaasFilename != null) {
title("JAAS");
File jaasFile = new File(jaasFilename);
println("JAAS file is defined in %s: %s",
SUN_SECURITY_JAAS_FILE, jaasFile);
failif(!jaasFile.exists(), CAT_JAAS,
"JAAS file does not exist: %s", jaasFile);
failif(!jaasFile.isFile(), CAT_JAAS,
"Specified JAAS file is not a file: %s", jaasFile);
}
}
/**
* Dump all tokens of a user
* @param user user
*/
public void dumpTokens(UserGroupInformation user) {
Collection<Token<? extends TokenIdentifier>> tokens
= user.getCredentials().getAllTokens();
title("Token Count: %d", tokens.size());
for (Token<? extends TokenIdentifier> token : tokens) {
println("Token %s", token.getKind());
}
}
/**
* Set the System property to true; return the old value for caching
* @param sysprop property
* @return the previous value
*/
private boolean getAndSet(String sysprop) {
boolean old = Boolean.getBoolean(sysprop);
System.setProperty(sysprop, "true");
return old;
}
/**
* Flush all active output channels, including {@Code System.err},
* so as to stay in sync with any JRE log messages.
*/
private void flush() {
if (out != null) {
out.flush();
} else {
System.out.flush();
}
System.err.flush();
}
/**
* Format and print a line of output.
* This goes to any output file, or
* is logged at info. The output is flushed before and after, to
* try and stay in sync with JRE logging.
* @param format format string
* @param args any arguments
*/
@VisibleForTesting
public void println(String format, Object... args) {
println(format(format, args));
}
/**
* Print a line of output. This goes to any output file, or
* is logged at info. The output is flushed before and after, to
* try and stay in sync with JRE logging.
* @param msg message string
*/
@VisibleForTesting
private void println(String msg) {
flush();
if (out != null) {
out.println(msg);
} else {
LOG.info(msg);
}
flush();
}
/**
* Print a title entry
* @param format format string
* @param args any arguments
*/
private void title(String format, Object... args) {
println("");
println("");
String msg = "== " + format(format, args) + " ==";
println(msg);
println("");
}
/**
* Print a system property, or {@link #UNSET} if unset.
* @param property property to print
*/
private void printSysprop(String property) {
println("%s = \"%s\"", property,
System.getProperty(property, UNSET));
}
/**
* Print a configuration option, or {@link #UNSET} if unset.
* @param option option to print
*/
private void printConfOpt(String option) {
println("%s = \"%s\"", option, conf.get(option, UNSET));
}
/**
* Print an environment variable's name and value; printing
* {@link #UNSET} if it is not set
* @param variable environment variable
*/
private void printEnv(String variable) {
String env = System.getenv(variable);
println("%s = \"%s\"", variable, env != null ? env : UNSET);
}
/**
* Dump any file to standard out; add a trailing newline
* @param file file to dump
* @throws IOException IO problems
*/
public void dump(File file) throws IOException {
try (FileInputStream in = new FileInputStream(file)) {
for (String line : IOUtils.readLines(in)) {
println("%s", line);
}
}
println("");
}
/**
* Format and raise a failure
*
* @param category category for exception
* @param message string formatting message
* @param args any arguments for the formatting
* @throws KerberosDiagsFailure containing the formatted text
*/
private void fail(String category, String message, Object... args)
throws KerberosDiagsFailure {
throw new KerberosDiagsFailure(category, message, args);
}
/**
* Conditional failure with string formatted arguments
* @param condition failure condition
* @param category category for exception
* @param message string formatting message
* @param args any arguments for the formatting
* @throws KerberosDiagsFailure containing the formatted text
* if the condition was met
*/
private void failif(boolean condition,
String category,
String message,
Object... args)
throws KerberosDiagsFailure {
if (condition) {
fail(category, message, args);
}
}
/**
* Format a string, treating a call where there are no varags values
* as a string to pass through unformatted.
* @param message message, which is either a format string + args, or
* a general string
* @param args argument array
* @return a string for printing.
*/
public static String format(String message, Object... args) {
if (args.length == 0) {
return message;
} else {
return String.format(message, args);
}
}
/**
* Diagnostics failures return the exit code 41, "unauthorized".
*
* They have a category, initially for testing: the category can be
* validated without having to match on the entire string.
*/
public static class KerberosDiagsFailure extends ExitUtil.ExitException {
private final String category;
public KerberosDiagsFailure(String category, String message) {
super(41, category + ": " + message);
this.category = category;
}
public KerberosDiagsFailure(String category, String message, Object... args) {
this(category, format(message, args));
}
public KerberosDiagsFailure(String category, Throwable throwable,
String message, Object... args) {
this(category, message, args);
initCause(throwable);
}
public String getCategory() {
return category;
}
}
}

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider;
import org.apache.slider.client.SliderClient;
import org.apache.slider.core.main.ServiceLauncher;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* This is just the entry point class
*/
public class Slider extends SliderClient {
public static final String SERVICE_CLASSNAME = "org.apache.slider.Slider";
/**
* This is the main entry point for the service launcher.
* @param args command line arguments.
*/
public static void main(String[] args) {
//turn the args to a list
List<String> argsList = Arrays.asList(args);
//create a new list, as the ArrayList type doesn't push() on an insert
List<String> extendedArgs = new ArrayList<String>(argsList);
//insert the service name
extendedArgs.add(0, SERVICE_CLASSNAME);
//now have the service launcher do its work
ServiceLauncher.serviceMain(extendedArgs);
}
}

795
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java Normal file
View File

@ -0,0 +1,795 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.SliderProviderFactory;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.SerializationConfig;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_HOME;
import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM;
/**
* Represents a cluster specification; designed to be sendable over the wire
* and persisted in JSON by way of Jackson.
*
* When used in cluster status operations the <code>info</code>
* and <code>statistics</code> maps contain information about the cluster.
*
* As a wire format it is less efficient in both xfer and ser/deser than
* a binary format, but by having one unified format for wire and persistence,
* the code paths are simplified.
*
* This was the original single-file specification/model used in the Hoya
* precursor to Slider. Its now retained primarily as a way to publish
* the current state of the application, or at least a fraction thereof ...
* the larger set of information from the REST API is beyond the scope of
* this structure.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ClusterDescription implements Cloneable {
protected static final Logger
log = LoggerFactory.getLogger(ClusterDescription.class);
private static final String UTF_8 = "UTF-8";
/**
* version counter
*/
public String version = "1.0";
/**
* Name of the cluster
*/
public String name;
/**
* Type of cluster
*/
public String type = SliderProviderFactory.DEFAULT_CLUSTER_TYPE;
/**
* State of the cluster
*/
public int state;
/*
State list for both clusters and nodes in them. Ordered so that destroyed follows
stopped.
Some of the states are only used for recording
the persistent state of the cluster and are not
seen in node descriptions
*/
/**
* Specification is incomplete & cannot
* be used: {@value}
*/
public static final int STATE_INCOMPLETE = StateValues.STATE_INCOMPLETE;
/**
* Spec has been submitted: {@value}
*/
public static final int STATE_SUBMITTED = StateValues.STATE_SUBMITTED;
/**
* Cluster created: {@value}
*/
public static final int STATE_CREATED = StateValues.STATE_CREATED;
/**
* Live: {@value}
*/
public static final int STATE_LIVE = StateValues.STATE_LIVE;
/**
* Stopped
*/
public static final int STATE_STOPPED = StateValues.STATE_STOPPED;
/**
* destroyed
*/
public static final int STATE_DESTROYED = StateValues.STATE_DESTROYED;
/**
* When was the cluster specification created?
* This is not the time a cluster was thawed; that will
* be in the <code>info</code> section.
*/
public long createTime;
/**
* When was the cluster specification last updated
*/
public long updateTime;
/**
* URL path to the original configuration
* files; these are re-read when
* restoring a cluster
*/
public String originConfigurationPath;
/**
* URL path to the generated configuration
*/
public String generatedConfigurationPath;
/**
* This is where the data goes
*/
public String dataPath;
/**
* cluster-specific options -to control both
* the Slider AM and the application that it deploys
*/
public Map<String, String> options = new HashMap<>();
/**
* cluster information
* This is only valid when querying the cluster status.
*/
public Map<String, String> info = new HashMap<>();
/**
* Statistics. This is only relevant when querying the cluster status
*/
public Map<String, Map<String, Integer>> statistics = new HashMap<>();
/**
* Instances: role->count
*/
public Map<String, List<String>> instances = new HashMap<>();
/**
* Role options,
* role -> option -> value
*/
public Map<String, Map<String, String>> roles = new HashMap<>();
/**
* List of key-value pairs to add to a client config to set up the client
*/
public Map<String, String> clientProperties = new HashMap<>();
/**
* Status information
*/
public Map<String, Object> status;
/**
* Liveness information; the same as returned
* on the <code>live/liveness/</code> URL
*/
public ApplicationLivenessInformation liveness;
/**
* Creator.
*/
public ClusterDescription() {
}
@Override
public String toString() {
try {
return toJsonString();
} catch (Exception e) {
log.debug("Failed to convert CD to JSON ", e);
return super.toString();
}
}
/**
* Shallow clone
* @return a shallow clone
* @throws CloneNotSupportedException
*/
@Override
public Object clone() throws CloneNotSupportedException {
return super.clone();
}
/**
* A deep clone of the spec. This is done inefficiently with a ser/derser
* @return the cluster description
*/
public ClusterDescription deepClone() {
try {
return fromJson(toJsonString());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Save a cluster description to a hadoop filesystem
* @param fs filesystem
* @param path path
* @param overwrite should any existing file be overwritten
* @throws IOException IO exception
*/
public void save(FileSystem fs, Path path, boolean overwrite) throws
IOException {
FSDataOutputStream dataOutputStream = fs.create(path, overwrite);
writeJsonAsBytes(dataOutputStream);
}
/**
* Save a cluster description to the local filesystem
* @param file file
* @throws IOException IO excpetion
*/
public void save(File file) throws IOException {
log.debug("Saving to {}", file.getAbsolutePath());
if (!file.getParentFile().mkdirs()) {
log.warn("Failed to mkdirs for {}", file.getParentFile());
}
DataOutputStream dataOutputStream = new DataOutputStream(new FileOutputStream(file));
writeJsonAsBytes(dataOutputStream);
}
/**
* Write the json as bytes -then close the file
* @param dataOutputStream an outout stream that will always be closed
* @throws IOException any failure
*/
private void writeJsonAsBytes(DataOutputStream dataOutputStream)
throws IOException {
try {
String json = toJsonString();
byte[] b = json.getBytes(UTF_8);
dataOutputStream.write(b);
} finally {
dataOutputStream.close();
}
}
/**
* Load from the filesystem
* @param fs filesystem
* @param path path
* @return a loaded CD
* @throws IOException IO problems
*/
public static ClusterDescription load(FileSystem fs, Path path)
throws IOException, JsonParseException, JsonMappingException {
FileStatus status = fs.getFileStatus(path);
byte[] b = new byte[(int) status.getLen()];
FSDataInputStream dataInputStream = fs.open(path);
int count = dataInputStream.read(b);
String json = new String(b, 0, count, UTF_8);
return fromJson(json);
}
/**
* Make a deep copy of the class
* @param source source
* @return the copy
*/
public static ClusterDescription copy(ClusterDescription source) {
//currently the copy is done by a generate/save. Inefficient but it goes
//down the tree nicely
try {
return fromJson(source.toJsonString());
} catch (IOException e) {
throw new RuntimeException("ClusterDescription copy failed " + e, e);
}
}
/**
* Convert to a JSON string
* @return a JSON string description
* @throws IOException Problems mapping/writing the object
*/
public String toJsonString() throws IOException,
JsonGenerationException,
JsonMappingException {
ObjectMapper mapper = new ObjectMapper();
mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
return mapper.writeValueAsString(this);
}
/**
* Convert from JSON
* @param json input
* @return the parsed JSON
* @throws IOException IO
* @throws JsonMappingException failure to map from the JSON to this class
*/
public static ClusterDescription fromJson(String json)
throws IOException, JsonParseException, JsonMappingException {
ObjectMapper mapper = new ObjectMapper();
try {
return mapper.readValue(json, ClusterDescription.class);
} catch (IOException e) {
log.error("Exception while parsing json : " + e + "\n" + json, e);
throw e;
}
}
/**
* Convert from input stream
* @param is input stream of cluster description
* @return the parsed JSON
* @throws IOException IO
* @throws JsonMappingException failure to map from the JSON to this class
*/
public static ClusterDescription fromStream(InputStream is)
throws IOException, JsonParseException, JsonMappingException {
if (is==null) {
throw new FileNotFoundException("Empty Stream");
}
ObjectMapper mapper = new ObjectMapper();
try {
return mapper.readValue(is, ClusterDescription.class);
} catch (IOException e) {
log.error("Exception while parsing input stream : {}", e, e);
throw e;
}
}
/**
* Convert from a JSON file
* @param jsonFile input file
* @return the parsed JSON
* @throws IOException IO problems
* @throws JsonMappingException failure to map from the JSON to this class
*/
public static ClusterDescription fromFile(File jsonFile)
throws IOException, JsonParseException, JsonMappingException {
ObjectMapper mapper = new ObjectMapper();
try {
return mapper.readValue(jsonFile, ClusterDescription.class);
} catch (IOException e) {
log.error("Exception while parsing json file {}" , jsonFile, e);
throw e;
}
}
/**
* Set a cluster option: a key val pair in the options {} section
* @param key key option name
* @param val value option value
*/
public void setOption(String key, String val) {
options.put(key, val);
}
/**
* Set a cluster option if it is unset. If it is already set,
* in the Cluster Description, it is left alone
* @param key key key to query/set
* @param val value value
*/
public void setOptionifUnset(String key, String val) {
if (options.get(key) == null) {
options.put(key, val);
}
}
/**
* Set an integer option -it's converted to a string before saving
* @param option option name
* @param val integer value
*/
public void setOption(String option, int val) {
setOption(option, Integer.toString(val));
}
/**
* Set a boolean option
* @param option option name
* @param val bool value
*/
public void setOption(String option, boolean val) {
setOption(option, Boolean.toString(val));
}
/**
* Get a cluster option or value
*
* @param key option key
* @param defVal option val
* @return resolved value or default
*/
public String getOption(String key, String defVal) {
String val = options.get(key);
return val != null ? val : defVal;
}
/**
* Get a cluster option or value
*
* @param key mandatory key
* @return the value
* @throws BadConfigException if the option is missing
*/
public String getMandatoryOption(String key) throws BadConfigException {
String val = options.get(key);
if (val == null) {
throw new BadConfigException("Missing option " + key);
}
return val;
}
/**
* Get an integer option; use {@link Integer#decode(String)} so as to take hex
* oct and bin values too.
*
* @param option option name
* @param defVal default value
* @return parsed value
* @throws NumberFormatException if the role could not be parsed.
*/
public int getOptionInt(String option, int defVal) {
String val = getOption(option, Integer.toString(defVal));
return Integer.decode(val);
}
/**
* Verify that an option is set: that is defined AND non-empty
* @param key key to verify
* @throws BadConfigException
*/
public void verifyOptionSet(String key) throws BadConfigException {
if (SliderUtils.isUnset(getOption(key, null))) {
throw new BadConfigException("Unset cluster option %s", key);
}
}
/**
* Get an option as a boolean. Note that {@link Boolean#valueOf(String)}
* is used for parsing -its policy of what is true vs false applies.
* @param option name
* @param defVal default
* @return the option.
*/
public boolean getOptionBool(String option, boolean defVal) {
return Boolean.valueOf(getOption(option, Boolean.toString(defVal)));
}
/**
* Get a role option
* @param role role to get from
* @param option option name
* @param defVal default value
* @return resolved value
*/
public String getRoleOpt(String role, String option, String defVal) {
Map<String, String> roleopts = getRole(role);
if (roleopts == null) {
return defVal;
}
String val = roleopts.get(option);
return val != null ? val : defVal;
}
/**
* Get a mandatory role option
* @param role role to get from
* @param option option name
* @return resolved value
* @throws BadConfigException if the option is not defined
*/
public String getMandatoryRoleOpt(String role, String option) throws
BadConfigException {
Map<String, String> roleopts = getRole(role);
if (roleopts == null) {
throw new BadConfigException("Missing role %s ", role);
}
String val = roleopts.get(option);
if (val == null) {
throw new BadConfigException("Missing option '%s' in role %s ", option, role);
}
return val;
}
/**
* Get a mandatory integer role option
* @param role role to get from
* @param option option name
* @return resolved value
* @throws BadConfigException if the option is not defined
*/
public int getMandatoryRoleOptInt(String role, String option)
throws BadConfigException {
getMandatoryRoleOpt(role, option);
return getRoleOptInt(role, option, 0);
}
/**
* look up a role and return its options
* @param role role
* @return role mapping or null
*/
public Map<String, String> getRole(String role) {
return roles.get(role);
}
/**
* Get a role -adding it to the roleopts map if
* none with that name exists
* @param role role
* @return role mapping
*/
public Map<String, String> getOrAddRole(String role) {
Map<String, String> map = getRole(role);
if (map == null) {
map = new HashMap<>();
}
roles.put(role, map);
return map;
}
/*
* return the Set of role names
*/
@JsonIgnore
public Set<String> getRoleNames() {
return new HashSet<>(roles.keySet());
}
/**
* Get a role whose presence is mandatory
* @param role role name
* @return the mapping
* @throws BadConfigException if the role is not there
*/
public Map<String, String> getMandatoryRole(String role) throws
BadConfigException {
Map<String, String> roleOptions = getRole(role);
if (roleOptions == null) {
throw new BadConfigException("Missing role " + role);
}
return roleOptions;
}
/**
* Get an integer role option; use {@link Integer#decode(String)} so as to take hex
* oct and bin values too.
*
* @param role role to get from
* @param option option name
* @param defVal default value
* @return parsed value
* @throws NumberFormatException if the role could not be parsed.
*/
public int getRoleOptInt(String role, String option, int defVal) {
String val = getRoleOpt(role, option, Integer.toString(defVal));
return Integer.decode(val);
}
/**
* Get an integer role option; use {@link Integer#decode(String)} so as to take hex
* oct and bin values too.
*
* @param role role to get from
* @param option option name
* @param defVal default value
* @return parsed value
* @throws NumberFormatException if the role could not be parsed.
*/
public long getRoleOptLong(String role, String option, long defVal) {
String val = getRoleOpt(role, option, Long.toString(defVal));
return Long.decode(val);
}
/**
* Set a role option, creating the role if necessary
* @param role role name
* @param option option name
* @param val value
*/
public void setRoleOpt(String role, String option, String val) {
Map<String, String> roleopts = getOrAddRole(role);
roleopts.put(option, val);
}
/**
* Set an integer role option, creating the role if necessary
* @param role role name
* @param option option name
* @param val integer value
*/
public void setRoleOpt(String role, String option, int val) {
setRoleOpt(role, option, Integer.toString(val));
}
/**
* Set a role option of any object, using its string value.
* This works for (Boxed) numeric values as well as other objects
* @param role role name
* @param option option name
* @param val non-null value
*/
public void setRoleOpt(String role, String option, Object val) {
setRoleOpt(role, option, val.toString());
}
/**
* Get the value of a role requirement (cores, RAM, etc).
* These are returned as integers, but there is special handling of the
* string {@link ResourceKeys#YARN_RESOURCE_MAX}, which triggers
* the return of the maximum value.
* @param role role to get from
* @param option option name
* @param defVal default value
* @param maxVal value to return if the max val is requested
* @return parsed value
* @throws NumberFormatException if the role could not be parsed.
*/
public int getRoleResourceRequirement(String role, String option, int defVal, int maxVal) {
String val = getRoleOpt(role, option, Integer.toString(defVal));
Integer intVal;
if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) {
intVal = maxVal;
} else {
intVal = Integer.decode(val);
}
return intVal;
}
/**
* Set the time for an information (human, machine) timestamp pair of fields.
* The human time is the time in millis converted via the {@link Date} class.
* @param keyHumanTime name of human time key
* @param keyMachineTime name of machine time
* @param time timestamp
*/
public void setInfoTime(String keyHumanTime, String keyMachineTime, long time) {
SliderUtils.setInfoTime(info, keyHumanTime, keyMachineTime, time);
}
/**
* Set an information string. This is content that is only valid in status
* reports.
* @param key key
* @param value string value
*/
@JsonIgnore
public void setInfo(String key, String value) {
info.put(key, value);
}
/**
* Get an information string. This is content that is only valid in status
* reports.
* @param key key
* @return the value or null
*/
@JsonIgnore
public String getInfo(String key) {
return info.get(key);
}
/**
* Get an information string. This is content that is only valid in status
* reports.
* @param key key
* @return the value or null
*/
@JsonIgnore
public boolean getInfoBool(String key) {
String val = info.get(key);
if (val != null) {
return Boolean.valueOf(val);
}
return false;
}
@JsonIgnore
public String getZkHosts() throws BadConfigException {
return getMandatoryOption(ZOOKEEPER_QUORUM);
}
/**
* Set the hosts for the ZK quorum
* @param zkHosts a comma separated list of hosts
*/
@JsonIgnore
public void setZkHosts(String zkHosts) {
setOption(ZOOKEEPER_QUORUM, zkHosts);
}
@JsonIgnore
public String getZkPath() throws BadConfigException {
return getMandatoryOption(ZOOKEEPER_PATH);
}
@JsonIgnore
public void setZkPath(String zkPath) {
setOption(ZOOKEEPER_PATH, zkPath);
}
/**
* HBase home: if non-empty defines where a copy of HBase is preinstalled
*/
@JsonIgnore
public String getApplicationHome() {
return getOption(INTERNAL_APPLICATION_HOME, "");
}
@JsonIgnore
public void setApplicationHome(String applicationHome) {
setOption(INTERNAL_APPLICATION_HOME, applicationHome);
}
/**
* The path in HDFS where the HBase image is
*/
@JsonIgnore
public String getImagePath() {
return getOption(INTERNAL_APPLICATION_IMAGE_PATH, "");
}
/**
* Set the path in HDFS where the HBase image is
*/
@JsonIgnore
public void setImagePath(String imagePath) {
setOption(INTERNAL_APPLICATION_IMAGE_PATH, imagePath);
}
/**
* Query for the image path being set (non null/non empty)
* @return true if there is a path in the image path option
*/
@JsonIgnore
public boolean isImagePathSet() {
return SliderUtils.isSet(getImagePath());
}
}

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionKeys.java Normal file
View File

@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api;
public class ClusterDescriptionKeys {
public static final String KEY_CLUSTER_LIVE = "live";
public static final String KEY_CLUSTER_FAILED = "failed";
}

93
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java Normal file
View File

@ -0,0 +1,93 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.SliderProviderFactory;
import java.util.Map;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM;
/**
* Operations on Cluster Descriptions
*/
public class ClusterDescriptionOperations {
public static ClusterDescription buildFromInstanceDefinition(AggregateConf aggregateConf) throws
BadConfigException {
ClusterDescription cd = new ClusterDescription();
aggregateConf.resolve();
//options are a merge of all globals
Map<String, String> options = cd.options;
SliderUtils.mergeMapsIgnoreDuplicateKeys(options,
aggregateConf.getInternal().global);
SliderUtils.mergeMapsIgnoreDuplicateKeys(options,
aggregateConf.getAppConf().global);
SliderUtils.mergeMapsIgnoreDuplicateKeys(options,
aggregateConf.getResources().global);
//roles are the role values merged in the same order
mergeInComponentMap(cd, aggregateConf.getInternal());
mergeInComponentMap(cd, aggregateConf.getAppConf());
mergeInComponentMap(cd, aggregateConf.getResources());
//now add the extra bits
cd.state = ClusterDescription.STATE_LIVE;
MapOperations internalOptions =
aggregateConf.getInternalOperations().getGlobalOptions();
MapOperations appOptions =
aggregateConf.getAppConfOperations().getGlobalOptions();
cd.type = internalOptions.getOption(InternalKeys.INTERNAL_PROVIDER_NAME,
SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
cd.dataPath = internalOptions.get(InternalKeys.INTERNAL_DATA_DIR_PATH);
cd.name = internalOptions.get(OptionKeys.APPLICATION_NAME);
cd.originConfigurationPath = internalOptions.get(InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH);
cd.generatedConfigurationPath = internalOptions.get(InternalKeys.INTERNAL_GENERATED_CONF_PATH);
cd.setImagePath(internalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH));
cd.setApplicationHome(internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME));
cd.setZkPath(appOptions.get(ZOOKEEPER_PATH));
cd.setZkHosts(appOptions.get(ZOOKEEPER_QUORUM));
return cd;
}
private static void mergeInComponentMap(ClusterDescription cd,
ConfTree confTree) {
Map<String, Map<String, String>> components = confTree.components;
for (Map.Entry<String, Map<String, String>> compEntry : components.entrySet()) {
String name = compEntry.getKey();
Map<String, String> destRole = cd.getOrAddRole(name);
Map<String, String> sourceComponent = compEntry.getValue();
SliderUtils.mergeMapsIgnoreDuplicateKeys(destRole, sourceComponent);
}
}
}

220
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterNode.java Normal file
View File

@ -0,0 +1,220 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.slider.api.proto.Messages;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
/**
* Describe a specific node in the cluster
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL )
public final class ClusterNode implements Cloneable {
protected static final Logger
LOG = LoggerFactory.getLogger(ClusterNode.class);
@JsonIgnore
public ContainerId containerId;
/**
* server name
*/
public String name;
/**
* UUID of container used in Slider RPC to refer to instances
*/
public String id;
public String role;
public int roleId;
public long createTime;
public long startTime;
/**
* flag set when it is released, to know if it has
* already been targeted for termination
*/
public boolean released;
public String host;
public String ip;
public String hostname;
public String hostUrl;
/**
* state from {@link ClusterDescription}
*/
public int state;
/**
* Exit code: only valid if the state >= STOPPED
*/
public int exitCode;
/**
* what was the command executed?
*/
public String command;
/**
* Any diagnostics
*/
public String diagnostics;
/**
* What is the tail output from the executed process (or [] if not started
* or the log cannot be picked up
*/
public String[] output;
/**
* Any environment details
*/
public String[] environment;
/**
* server-side ctor takes the container ID and builds the name from it
* @param containerId container ID; can be null
*/
public ClusterNode(ContainerId containerId) {
if (containerId != null) {
this.containerId = containerId;
this.name = containerId.toString();
}
}
/**
* ctor for deserialization
*/
public ClusterNode() {
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append(name).append(": ");
builder.append(state).append("\n");
builder.append("state: ").append(state).append("\n");
builder.append("role: ").append(role).append("\n");
append(builder, "host", host);
append(builder, "hostURL", hostUrl);
append(builder, "command", command);
if (output != null) {
for (String line : output) {
builder.append(line).append("\n");
}
}
append(builder, "diagnostics", diagnostics);
return builder.toString();
}
private void append(StringBuilder builder, String key, Object val) {
if (val != null) {
builder.append(key).append(": ").append(val.toString()).append("\n");
}
}
/**
* Convert to a JSON string
* @return a JSON string description
* @throws IOException Problems mapping/writing the object
*/
public String toJsonString() throws IOException {
ObjectMapper mapper = new ObjectMapper();
return mapper.writeValueAsString(this);
}
/**
* Convert from JSON
* @param json input
* @return the parsed JSON
* @throws IOException IO
*/
public static ClusterNode fromJson(String json)
throws IOException, JsonParseException, JsonMappingException {
ObjectMapper mapper = new ObjectMapper();
try {
return mapper.readValue(json, ClusterNode.class);
} catch (IOException e) {
LOG.error("Exception while parsing json : {}\n{}", e , json, e);
throw e;
}
}
/**
* Build from a protobuf response
* @param message
* @return the deserialized node
*/
public static ClusterNode fromProtobuf(Messages.RoleInstanceState message) {
ClusterNode node = new ClusterNode();
node.name = message.getName();
node.command = message.getCommand();
node.diagnostics = message.getDiagnostics();
String[] arr;
int environmentCount = message.getEnvironmentCount();
if (environmentCount > 0) {
arr = new String[environmentCount];
node.environment = message.getEnvironmentList().toArray(arr);
}
node.exitCode = message.getExitCode();
int outputCount = message.getOutputCount();
if (outputCount > 0) {
arr = new String[outputCount];
node.output = message.getOutputList().toArray(arr);
}
node.role = message.getRole();
node.roleId = message.getRoleId();
node.state = message.getState();
node.host = message.getHost();
node.hostUrl = message.getHostURL();
node.createTime = message.getCreateTime();
node.startTime = message.getStartTime();
node.released = message.getReleased();
return node;
}
@Override
public Object clone() throws CloneNotSupportedException {
return super.clone();
}
public ClusterNode doClone() {
try {
return (ClusterNode)clone();
} catch (CloneNotSupportedException e) {
//not going to happen. This is a final class
return null;
}
}
}

199
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java Normal file
View File

@ -0,0 +1,199 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api;
/**
* Keys for internal use, go into `internal.json` and not intended for normal
* use except when tuning Slider AM operations
*/
public interface InternalKeys {
/**
* Home dir of the app: {@value}
* If set, implies there is a home dir to use
*/
String INTERNAL_APPLICATION_HOME = "internal.application.home";
/**
* Path to an image file containing the app: {@value}
*/
String INTERNAL_APPLICATION_IMAGE_PATH = "internal.application.image.path";
/**
* Time in milliseconds to wait after forking any in-AM
* process before attempting to start up the containers: {@value}
*
* A shorter value brings the cluster up faster, but means that if the
* in AM process fails (due to a bad configuration), then time
* is wasted starting containers on a cluster that isn't going to come
* up
*/
String INTERNAL_CONTAINER_STARTUP_DELAY = "internal.container.startup.delay";
/**
* internal temp directory: {@value}
*/
String INTERNAL_AM_TMP_DIR = "internal.am.tmp.dir";
/**
* internal temp directory: {@value}
*/
String INTERNAL_TMP_DIR = "internal.tmp.dir";
/**
* where a snapshot of the original conf dir is: {@value}
*/
String INTERNAL_SNAPSHOT_CONF_PATH = "internal.snapshot.conf.path";
/**
* where a snapshot of the original conf dir is: {@value}
*/
String INTERNAL_GENERATED_CONF_PATH = "internal.generated.conf.path";
/**
* where a snapshot of the original conf dir is: {@value}
*/
String INTERNAL_PROVIDER_NAME = "internal.provider.name";
/**
* where a snapshot of the original conf dir is: {@value}
*/
String INTERNAL_DATA_DIR_PATH = "internal.data.dir.path";
/**
* where the app def is stored
*/
String INTERNAL_APPDEF_DIR_PATH = "internal.appdef.dir.path";
/**
* where addons for the app are stored
*/
String INTERNAL_ADDONS_DIR_PATH = "internal.addons.dir.path";
/**
* Time in milliseconds to wait after forking any in-AM
* process before attempting to start up the containers: {@value}
*
* A shorter value brings the cluster up faster, but means that if the
* in AM process fails (due to a bad configuration), then time
* is wasted starting containers on a cluster that isn't going to come
* up
*/
int DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY = 5000;
/**
* Time in seconds before a container is considered long-lived.
* Shortlived containers are interpreted as a problem with the role
* and/or the host: {@value}
*/
String INTERNAL_CONTAINER_FAILURE_SHORTLIFE =
"internal.container.failure.shortlife";
/**
* Default short life threshold: {@value}
*/
int DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE = 60;
/**
* Version of the app: {@value}
*/
String KEYTAB_LOCATION = "internal.keytab.location";
/**
* Queue used to deploy the app: {@value}
*/
String INTERNAL_QUEUE = "internal.queue";
/**
* Flag to indicate whether or not the chaos monkey is enabled:
* {@value}
*/
String CHAOS_MONKEY_ENABLED = "internal.chaos.monkey.enabled";
boolean DEFAULT_CHAOS_MONKEY_ENABLED = false;
/**
* Rate
*/
String CHAOS_MONKEY_INTERVAL = "internal.chaos.monkey.interval";
String CHAOS_MONKEY_INTERVAL_DAYS = CHAOS_MONKEY_INTERVAL + ".days";
String CHAOS_MONKEY_INTERVAL_HOURS = CHAOS_MONKEY_INTERVAL + ".hours";
String CHAOS_MONKEY_INTERVAL_MINUTES = CHAOS_MONKEY_INTERVAL + ".minutes";
String CHAOS_MONKEY_INTERVAL_SECONDS = CHAOS_MONKEY_INTERVAL + ".seconds";
int DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS = 0;
int DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0;
int DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0;
String CHAOS_MONKEY_DELAY = "internal.chaos.monkey.delay";
String CHAOS_MONKEY_DELAY_DAYS = CHAOS_MONKEY_DELAY + ".days";
String CHAOS_MONKEY_DELAY_HOURS = CHAOS_MONKEY_DELAY + ".hours";
String CHAOS_MONKEY_DELAY_MINUTES = CHAOS_MONKEY_DELAY + ".minutes";
String CHAOS_MONKEY_DELAY_SECONDS = CHAOS_MONKEY_DELAY + ".seconds";
int DEFAULT_CHAOS_MONKEY_STARTUP_DELAY = 0;
/**
* Prefix for all chaos monkey probabilities
*/
String CHAOS_MONKEY_PROBABILITY =
"internal.chaos.monkey.probability";
/**
* Probabilies are out of 10000 ; 100==1%
*/
/**
* Probability of a monkey check killing the AM: {@value}
*/
String CHAOS_MONKEY_PROBABILITY_AM_FAILURE =
CHAOS_MONKEY_PROBABILITY + ".amfailure";
/**
* Default probability of a monkey check killing the AM: {@value}
*/
int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 0;
/**
* Probability of a monkey check killing the AM: {@value}
*/
String CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE =
CHAOS_MONKEY_PROBABILITY + ".amlaunchfailure";
/**
* Probability of a monkey check killing a container: {@value}
*/
String CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE =
CHAOS_MONKEY_PROBABILITY + ".containerfailure";
/**
* Default probability of a monkey check killing the a container: {@value}
*/
int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 0;
/**
* 1% of chaos
*/
int PROBABILITY_PERCENT_1 = 100;
/**
* 100% for chaos values
*/
int PROBABILITY_PERCENT_100 = 100 * PROBABILITY_PERCENT_1;
/**
* interval between checks for escalation: {@value}
*/
String ESCALATION_CHECK_INTERVAL = "escalation.check.interval.seconds";
/**
* default value: {@value}
*/
int DEFAULT_ESCALATION_CHECK_INTERVAL = 30;
}

58
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api;
/**
* Keys for entries in the <code>options</code> section
* of a cluster description.
*/
public interface OptionKeys extends InternalKeys {
/**
* Time in milliseconds to wait after forking any in-AM
* process before attempting to start up the containers: {@value}
*
* A shorter value brings the cluster up faster, but means that if the
* in AM process fails (due to a bad configuration), then time
* is wasted starting containers on a cluster that isn't going to come
* up
*/
String APPLICATION_TYPE = "application.type";
String APPLICATION_NAME = "application.name";
/**
* Prefix for site.xml options: {@value}
*/
String SITE_XML_PREFIX = "site.";
/**
* Zookeeper quorum host list: {@value}
*/
String ZOOKEEPER_QUORUM = "zookeeper.quorum";
String ZOOKEEPER_HOSTS = "zookeeper.hosts";
String ZOOKEEPER_PORT = "zookeeper.port";
/**
* Zookeeper path value (string): {@value}
*/
String ZOOKEEPER_PATH = "zookeeper.path";
}

201
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java Normal file
View File

@ -0,0 +1,201 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api;
/**
* These are the keys valid in resource options
*
/*
Container failure window.
The window is calculated in minutes as as (days * 24 *60 + hours* 24 + minutes)
Every interval of this period after the AM is started/restarted becomes
the time period in which the CONTAINER_FAILURE_THRESHOLD value is calculated.
After the window limit is reached, the failure counts are reset. This
is not a sliding window/moving average policy, simply a rule such as
"every six hours the failure count is reset"
<pre>
===========================================================================
</pre>
*/
public interface ResourceKeys {
/**
* #of instances of a component: {@value}
*
*/
String COMPONENT_INSTANCES = "yarn.component.instances";
/**
* Whether to use unique names for each instance of a component: {@value}
*/
String UNIQUE_NAMES = "component.unique.names";
/**
* Amount of memory to ask YARN for in MB.
* <i>Important:</i> this may be a hard limit on the
* amount of RAM that the service can use
* {@value}
*/
String YARN_MEMORY = "yarn.memory";
/** {@value} */
int DEF_YARN_MEMORY = 256;
/**
* Number of cores/virtual cores to ask YARN for
* {@value}
*/
String YARN_CORES = "yarn.vcores";
/**
* Number of disks per instance to ask YARN for
* {@value}
*/
String YARN_DISKS = "yarn.disks.count-per-instance";
/**
* Disk size per disk to ask YARN for
* {@value}
*/
String YARN_DISK_SIZE = "yarn.disk.size";
/** {@value} */
int DEF_YARN_CORES = 1;
/**
* Label expression that this container must satisfy
* {@value}
*/
String YARN_LABEL_EXPRESSION = "yarn.label.expression";
/** default label expression: */
String DEF_YARN_LABEL_EXPRESSION = null;
/**
* Constant to indicate that the requirements of a YARN resource limit
* (cores, memory, ...) should be set to the maximum allowed by
* the queue into which the YARN container requests are placed.
*/
String YARN_RESOURCE_MAX = "max";
/**
* Mandatory property for all roles
* 1. this must be defined.
* 2. this must be >= 1
* 3. this must not match any other role priority in the cluster.
*/
String COMPONENT_PRIORITY = "yarn.role.priority";
/**
* placement policy
*/
String COMPONENT_PLACEMENT_POLICY = "yarn.component.placement.policy";
/**
* Maximum number of node failures that can be tolerated by a component on a specific node
*/
String NODE_FAILURE_THRESHOLD =
"yarn.node.failure.threshold";
/**
* maximum number of failed containers (in a single role)
* before the cluster is deemed to have failed {@value}
*/
String CONTAINER_FAILURE_THRESHOLD =
"yarn.container.failure.threshold";
/**
* prefix for the time of the container failure reset window.
* {@value}
*/
String CONTAINER_FAILURE_WINDOW =
"yarn.container.failure.window";
int DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS = 0;
int DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS = 6;
int DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES = 0;
/**
* Default failure threshold: {@value}
*/
int DEFAULT_CONTAINER_FAILURE_THRESHOLD = 5;
/**
* Default node failure threshold for a component instance: {@value}
* Should to be lower than default component failure threshold to allow
* the component to start elsewhere
*/
int DEFAULT_NODE_FAILURE_THRESHOLD = 3;
/**
* Failure threshold is unlimited: {@value}
*/
int NODE_FAILURE_THRESHOLD_UNLIMITED = -1;
/**
* Time in seconds to escalate placement delay
*/
String PLACEMENT_ESCALATE_DELAY =
"yarn.placement.escalate.seconds";
/**
* Time to have a strict placement policy outstanding before
* downgrading to a lax placement (for those components which permit that).
* <ol>
* <li>For strictly placed components, there's no relaxation.</li>
* <li>For components with no locality, there's no need to relax</li>
* </ol>
*
*/
int DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS = 30;
/**
* Log aggregation include, exclude patterns
*/
String YARN_LOG_INCLUDE_PATTERNS = "yarn.log.include.patterns";
String YARN_LOG_EXCLUDE_PATTERNS = "yarn.log.exclude.patterns";
String YARN_PROFILE_NAME = "yarn.resource-profile-name";
/**
* Window of time where application master's failure count
* can be reset to 0.
*/
String YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS =
"yarn.resourcemanager.am.retry-count-window-ms";
/**
* The default window for Slider.
*/
long DEFAULT_AM_RETRY_COUNT_WINDOW_MS = 300000;
}

116
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/RoleKeys.java Normal file
View File

@ -0,0 +1,116 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api;
/**
* Standard options for roles
*/
public interface RoleKeys {
/**
* The name of a role: {@value}
*/
String ROLE_NAME = "role.name";
/**
* The group of a role: {@value}
*/
String ROLE_GROUP = "role.group";
/**
* Status report: number actually granted : {@value}
*/
String ROLE_ACTUAL_INSTANCES = "role.actual.instances";
/**
* Status report: number currently requested: {@value}
*/
String ROLE_REQUESTED_INSTANCES = "role.requested.instances";
/**
* Status report: number currently being released: {@value}
*/
String ROLE_RELEASING_INSTANCES = "role.releasing.instances";
/**
* Status report: total number that have failed: {@value}
*/
String ROLE_FAILED_INSTANCES = "role.failed.instances";
/**
* Status report: number that have failed recently: {@value}
*/
String ROLE_FAILED_RECENTLY_INSTANCES = "role.failed.recently.instances";
/**
* Status report: number that have failed for node-related issues: {@value}
*/
String ROLE_NODE_FAILED_INSTANCES = "role.failed.node.instances";
/**
* Status report: number that been pre-empted: {@value}
*/
String ROLE_PREEMPTED_INSTANCES = "role.failed.preempted.instances";
/**
* Number of pending anti-affine instances: {@value}
*/
String ROLE_PENDING_AA_INSTANCES = "role.pending.aa.instances";
/**
* Status report: number currently being released: {@value}
*/
String ROLE_FAILED_STARTING_INSTANCES = "role.failed.starting.instances";
/**
* Extra arguments (non-JVM) to use when starting this role
*/
String ROLE_ADDITIONAL_ARGS = "role.additional.args";
/**
* JVM heap size for Java applications in MB. Only relevant for Java applications.
* This MUST be less than or equal to the {@link ResourceKeys#YARN_MEMORY} option
* {@value}
*/
String JVM_HEAP = "jvm.heapsize";
/*
* GC options for Java applications.
*/
String GC_OPTS = "gc.opts";
/**
* JVM options other than heap size. Only relevant for Java applications.
* {@value}
*/
String JVM_OPTS = "jvm.opts";
/**
* All keys w/ env. are converted into env variables and passed down
*/
String ENV_PREFIX = "env.";
/**
* Container service record attribute prefix.
*/
String SERVICE_RECORD_ATTRIBUTE_PREFIX = "service.record.attribute";
}

167
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java Normal file
View File

@ -0,0 +1,167 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.types.NodeInformation;
import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.PingInformation;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
import java.io.IOException;
import java.util.Map;
/**
* API exported by the slider remote REST/IPC endpoints.
*/
public interface SliderApplicationApi {
/**
* Get the aggregate desired model
* @return the aggregate configuration of what was asked for
* -before resolution has taken place
* @throws IOException on any failure
*/
AggregateConf getDesiredModel() throws IOException;
/**
* Get the desired application configuration
* @return the application configuration asked for
* -before resolution has taken place
* @throws IOException on any failure
*/
ConfTreeOperations getDesiredAppconf() throws IOException;
/**
* Get the desired YARN resources
* @return the resources asked for
* -before resolution has taken place
* @throws IOException on any failure
*/
ConfTreeOperations getDesiredResources() throws IOException;
/**
* Put an updated resources structure. This triggers a cluster flex
* operation
* @param updated updated resources
* @throws IOException on any problem.
*/
void putDesiredResources(ConfTree updated) throws IOException;
/**
* Get the aggregate resolved model
* @return the aggregate configuration of what was asked for
* -after resolution has taken place
* @throws IOException on any failure
*/
AggregateConf getResolvedModel() throws IOException;
/**
* Get the resolved application configuration
* @return the application configuration asked for
* -after resolution has taken place
* @throws IOException on any failure
*/
ConfTreeOperations getResolvedAppconf() throws IOException;
/**
* Get the resolved YARN resources
* @return the resources asked for
* -after resolution has taken place
* @throws IOException on any failure
*/
ConfTreeOperations getResolvedResources() throws IOException;
/**
* Get the live YARN resources
* @return the live set of resources in the cluster
* @throws IOException on any failure
*/
ConfTreeOperations getLiveResources() throws IOException;
/**
* Get a map of live containers [containerId:info]
* @return a possibly empty list of serialized containers
* @throws IOException on any failure
*/
Map<String, ContainerInformation> enumContainers() throws IOException;
/**
* Get a container from the container Id
* @param containerId YARN container ID
* @return the container information
* @throws IOException on any failure
*/
ContainerInformation getContainer(String containerId) throws IOException;
/**
* List all components into a map of [name:info]
* @return a possibly empty map of components
* @throws IOException on any failure
*/
Map<String, ComponentInformation> enumComponents() throws IOException;
/**
* Get information about a component
* @param componentName name of the component
* @return the component details
* @throws IOException on any failure
*/
ComponentInformation getComponent(String componentName) throws IOException;
/**
* List all nodes into a map of [name:info]
* @return a possibly empty list of nodes
* @throws IOException on any failure
*/
NodeInformationList getLiveNodes() throws IOException;
/**
* Get information about a node
* @param hostname name of the node
* @return the node details
* @throws IOException on any failure
*/
NodeInformation getLiveNode(String hostname) throws IOException;
/**
* Ping as a GET
* @param text text to include
* @return the response
* @throws IOException on any failure
*/
PingInformation ping(String text) throws IOException;
/**
* Stop the AM (async operation)
* @param text text to include
* @throws IOException on any failure
*/
void stop(String text) throws IOException;
/**
* Get the application liveness
* @return current liveness information
* @throws IOException
*/
ApplicationLivenessInformation getApplicationLiveness() throws IOException;
}

179
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java Normal file
View File

@ -0,0 +1,179 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License") throws IOException, YarnException; you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api;
import org.apache.hadoop.ipc.VersionedProtocol;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.common.SliderXmlConfKeys;
import java.io.IOException;
/**
* Cluster protocol. This can currently act as a versioned IPC
* endpoint or be relayed via protobuf
*/
@KerberosInfo(serverPrincipal = SliderXmlConfKeys.KEY_KERBEROS_PRINCIPAL)
public interface SliderClusterProtocol extends VersionedProtocol {
long versionID = 0x01;
/**
* Stop the cluster
*/
Messages.StopClusterResponseProto stopCluster(Messages.StopClusterRequestProto request) throws
IOException, YarnException;
/**
* Upgrade the application containers
*
* @param request upgrade containers request object
* @return upgrade containers response object
* @throws IOException
* @throws YarnException
*/
Messages.UpgradeContainersResponseProto upgradeContainers(
Messages.UpgradeContainersRequestProto request) throws IOException,
YarnException;
/**
* Flex the cluster.
*/
Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request)
throws IOException;
/**
* Get the current cluster status
*/
Messages.GetJSONClusterStatusResponseProto getJSONClusterStatus(Messages.GetJSONClusterStatusRequestProto request)
throws IOException, YarnException;
/**
* List all running nodes in a role
*/
Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(Messages.ListNodeUUIDsByRoleRequestProto request)
throws IOException, YarnException;
/**
* Get the details on a node
*/
Messages.GetNodeResponseProto getNode(Messages.GetNodeRequestProto request)
throws IOException, YarnException;
/**
* Get the
* details on a list of nodes.
* Unknown nodes are not returned
* <i>Important: the order of the results are undefined</i>
*/
Messages.GetClusterNodesResponseProto getClusterNodes(Messages.GetClusterNodesRequestProto request)
throws IOException, YarnException;
/**
* Echo back the submitted text (after logging it).
* Useful for adding information to the log, and for testing round trip
* operations of the protocol
* @param request request
* @return response
* @throws IOException
* @throws YarnException
*/
Messages.EchoResponseProto echo(Messages.EchoRequestProto request) throws IOException, YarnException;
/**
* Kill an identified container
* @param request request containing the container to kill
* @return the response
* @throws IOException
* @throws YarnException
*/
Messages.KillContainerResponseProto killContainer(Messages.KillContainerRequestProto request)
throws IOException, YarnException;
/**
* AM to commit suicide. If the Hadoop halt entry point has not been disabled,
* this will fail rather than return with a response.
* @param request request
* @return response (this is not the expected outcome)
* @throws IOException
* @throws YarnException
*/
Messages.AMSuicideResponseProto amSuicide(Messages.AMSuicideRequestProto request)
throws IOException;
/**
* Get the instance definition
*/
Messages.GetInstanceDefinitionResponseProto getInstanceDefinition(
Messages.GetInstanceDefinitionRequestProto request)
throws IOException, YarnException;
/**
* Get the application liveness
* @return current liveness information
* @throws IOException
*/
Messages.ApplicationLivenessInformationProto getLivenessInformation(
Messages.GetApplicationLivenessRequestProto request
) throws IOException;
Messages.GetLiveContainersResponseProto getLiveContainers(
Messages.GetLiveContainersRequestProto request
) throws IOException;
Messages.ContainerInformationProto getLiveContainer(
Messages.GetLiveContainerRequestProto request
) throws IOException;
Messages.GetLiveComponentsResponseProto getLiveComponents(
Messages.GetLiveComponentsRequestProto request
) throws IOException;
Messages.ComponentInformationProto getLiveComponent(
Messages.GetLiveComponentRequestProto request
) throws IOException;
Messages.GetLiveNodesResponseProto getLiveNodes(
Messages.GetLiveNodesRequestProto request
) throws IOException;
Messages.NodeInformationProto getLiveNode(
Messages.GetLiveNodeRequestProto request
) throws IOException;
Messages.WrappedJsonProto getModelDesired(Messages.EmptyPayloadProto request) throws IOException;
Messages.WrappedJsonProto getModelDesiredAppconf(Messages.EmptyPayloadProto request) throws IOException;
Messages.WrappedJsonProto getModelDesiredResources(Messages.EmptyPayloadProto request) throws IOException;
Messages.WrappedJsonProto getModelResolved(Messages.EmptyPayloadProto request) throws IOException;
Messages.WrappedJsonProto getModelResolvedAppconf(Messages.EmptyPayloadProto request) throws IOException;
Messages.WrappedJsonProto getModelResolvedResources(Messages.EmptyPayloadProto request) throws IOException;
Messages.WrappedJsonProto getLiveResources(Messages.EmptyPayloadProto request) throws IOException;
Messages.GetCertificateStoreResponseProto getClientCertificateStore(Messages.GetCertificateStoreRequestProto request)
throws IOException;
}

53
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/StateValues.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api;
/**
* Enumeration of state values
*/
public class StateValues {
/**
* Specification is incomplete & cannot
* be used: {@value}
*/
public static final int STATE_INCOMPLETE = 0;
/**
* Spec has been submitted: {@value}
*/
public static final int STATE_SUBMITTED = 1;
/**
* Cluster created: {@value}
*/
public static final int STATE_CREATED = 2;
/**
* Live: {@value}
*/
public static final int STATE_LIVE = 3;
/**
* Stopped
*/
public static final int STATE_STOPPED = 4;
/**
* destroyed
*/
public static final int STATE_DESTROYED = 5;
}

117
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/StatusKeys.java Normal file
View File

@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api;
import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
/**
* Contains status and statistics keys
*/
public interface StatusKeys {
String STATISTICS_CONTAINERS_ACTIVE_REQUESTS = "containers.active.requests";
String STATISTICS_CONTAINERS_COMPLETED = "containers.completed";
String STATISTICS_CONTAINERS_DESIRED = "containers.desired";
String STATISTICS_CONTAINERS_FAILED = "containers.failed";
String STATISTICS_CONTAINERS_FAILED_RECENTLY = "containers.failed.recently";
String STATISTICS_CONTAINERS_FAILED_NODE = "containers.failed.node";
String STATISTICS_CONTAINERS_PREEMPTED = "containers.failed.preempted";
String STATISTICS_CONTAINERS_LIVE = "containers.live";
String STATISTICS_CONTAINERS_REQUESTED = "containers.requested";
String STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING = "containers.anti-affine.pending";
String STATISTICS_CONTAINERS_STARTED = "containers.start.started";
String STATISTICS_CONTAINERS_START_FAILED =
"containers.start.failed";
String STATISTICS_CONTAINERS_SURPLUS =
"containers.surplus";
String STATISTICS_CONTAINERS_UNKNOWN_COMPLETED =
"containers.unknown.completed";
/**
* No of containers provided on AM restart
*/
String INFO_CONTAINERS_AM_RESTART = "containers.at.am-restart";
String INFO_CREATE_TIME_MILLIS = "create.time.millis";
String INFO_CREATE_TIME_HUMAN = "create.time";
String INFO_LIVE_TIME_MILLIS = "live.time.millis";
String INFO_LIVE_TIME_HUMAN = "live.time";
String INFO_FLEX_TIME_MILLIS = "flex.time.millis";
String INFO_FLEX_TIME_HUMAN = "flex.time";
String INFO_MASTER_ADDRESS = "info.master.address";
/**
* System time in millis when the status report was generated
*/
String INFO_STATUS_TIME_MILLIS = "status.time.millis";
/**
* System time in human form when the status report was generated
*/
String INFO_STATUS_TIME_HUMAN = "status.time";
String INFO_AM_APP_ID = "info.am.app.id";
String INFO_AM_ATTEMPT_ID = "info.am.attempt.id";
String INFO_AM_CONTAINER_ID = "info.am.container.id";
String INFO_AM_HOSTNAME = "info.am.hostname";
String INFO_AM_RPC_PORT = "info.am.rpc.port";
String INFO_AM_WEB_PORT = "info.am.web.port";
String INFO_AM_WEB_URL = "info.am.web.url";
String INFO_AM_AGENT_STATUS_PORT = "info.am.agent.status.port";
String INFO_AM_AGENT_OPS_PORT = "info.am.agent.ops.port";
String INFO_AM_AGENT_OPS_URL = "info.am.agent.ops.url";
String INFO_AM_AGENT_STATUS_URL = "info.am.agent.status.url";
/**
* info: #of instances of a component requested: {@value}
*
*/
String COMPONENT_INSTANCES_ACTUAL = COMPONENT_INSTANCES + ".actual";
/**
* info: #of instances of a component requested: {@value}
*
*/
String COMPONENT_INSTANCES_REQUESTING = COMPONENT_INSTANCES + ".requesting";
/**
* info: #of instances of a component being released: {@value}
*
*/
String COMPONENT_INSTANCES_RELEASING = COMPONENT_INSTANCES + ".releasing";
/**
* info: #of instances of a component failed: {@value}
*
*/
String COMPONENT_INSTANCES_FAILED = COMPONENT_INSTANCES + ".failed";
/**
* info: #of instances of a component started: {@value}
*
*/
String COMPONENT_INSTANCES_STARTED = COMPONENT_INSTANCES + ".started";
/**
* info: #of instances of a component completed: {@value}
*
*/
String COMPONENT_INSTANCES_COMPLETED = COMPONENT_INSTANCES + ".completed";
}

34473
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java Normal file
View File

File diff suppressed because it is too large Load Diff

319
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java Normal file
View File

@ -0,0 +1,319 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api.proto;
import com.google.protobuf.ByteString;
import org.apache.commons.io.IOUtils;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.types.NodeEntryInformation;
import org.apache.slider.api.types.NodeInformation;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.persist.AggregateConfSerDeser;
import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.apache.slider.server.services.security.SecurityStore;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Class to handle marshalling of REST
* types to/from Protobuf records.
*/
public class RestTypeMarshalling {
public static Messages.ApplicationLivenessInformationProto
marshall(ApplicationLivenessInformation info) {
Messages.ApplicationLivenessInformationProto.Builder builder =
Messages.ApplicationLivenessInformationProto.newBuilder();
builder.setAllRequestsSatisfied(info.allRequestsSatisfied);
builder.setRequestsOutstanding(info.requestsOutstanding);
return builder.build();
}
public static ApplicationLivenessInformation
unmarshall(Messages.ApplicationLivenessInformationProto wire) {
ApplicationLivenessInformation info = new ApplicationLivenessInformation();
info.allRequestsSatisfied = wire.getAllRequestsSatisfied();
info.requestsOutstanding = wire.getRequestsOutstanding();
return info;
}
public static ComponentInformation
unmarshall(Messages.ComponentInformationProto wire) {
ComponentInformation info = new ComponentInformation();
info.name = wire.getName();
info.priority = wire.getPriority();
info.placementPolicy = wire.getPlacementPolicy();
info.actual = wire.getActual();
info.completed = wire.getCompleted();
info.desired = wire.getDesired();
info.failed = wire.getFailed();
info.releasing = wire.getReleasing();
info.requested = wire.getRequested();
info.started = wire.getStarted();
info.startFailed = wire.getStartFailed();
info.totalRequested = wire.getTotalRequested();
info.containers = new ArrayList<>(wire.getContainersList());
if (wire.hasFailureMessage()) {
info.failureMessage = wire.getFailureMessage();
}
if (wire.hasPendingAntiAffineRequestCount()) {
info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
}
if (wire.hasIsAARequestOutstanding()) {
info.isAARequestOutstanding = wire.getIsAARequestOutstanding();
}
return info;
}
public static Messages.GetCertificateStoreResponseProto marshall(
SecurityStore securityStore) throws IOException {
Messages.GetCertificateStoreResponseProto.Builder builder =
Messages.GetCertificateStoreResponseProto.newBuilder();
builder.setStore(ByteString.copyFrom(getStoreBytes(securityStore)));
return builder.build();
}
private static byte[] getStoreBytes(SecurityStore securityStore)
throws IOException {
InputStream is = null;
byte[] storeBytes;
try {
is = new FileInputStream(securityStore.getFile());
storeBytes = IOUtils.toByteArray(is);
} finally {
if (is != null) {
is.close();
}
}
return storeBytes;
}
public static byte[] unmarshall(Messages.GetCertificateStoreResponseProto response) {
return response.getStore().toByteArray();
}
public static Messages.ComponentInformationProto marshall(ComponentInformation info) {
Messages.ComponentInformationProto.Builder builder =
Messages.ComponentInformationProto.newBuilder();
builder.setName(info.name);
builder.setPriority(info.priority);
builder.setPlacementPolicy(info.placementPolicy);
builder.setActual(info.actual);
builder.setCompleted(info.completed);
builder.setDesired(info.desired);
builder.setFailed(info.failed);
builder.setReleasing(info.releasing);
builder.setRequested(info.requested);
builder.setStarted(info.started);
builder.setStartFailed(info.startFailed);
builder.setTotalRequested(info.totalRequested);
builder.setNodeFailed(info.nodeFailed);
builder.setPreempted(info.preempted);
builder.setFailedRecently(info.failedRecently);
if (info.failureMessage != null) {
builder.setFailureMessage(info.failureMessage);
}
if (info.containers != null) {
builder.addAllContainers(info.containers);
}
builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount);
builder.setIsAARequestOutstanding(info.isAARequestOutstanding);
return builder.build();
}
public static Messages.NodeInformationProto marshall(NodeInformation info) {
Messages.NodeInformationProto.Builder builder =
Messages.NodeInformationProto.newBuilder();
builder.setHostname(info.hostname);
builder.setLastUpdated(info.lastUpdated);
builder.setState(info.state != null? info.state : "unknown");
builder.setRackName(info.rackName != null ? info.rackName : "");
builder.setHealthReport(info.healthReport != null ? info.healthReport : "");
builder.setHttpAddress(info.httpAddress != null ? info.httpAddress : "");
builder.setLabels(info.labels != null ? info.labels: "");
if (info.entries != null) {
for (Map.Entry<String, NodeEntryInformation> elt : info.entries.entrySet()) {
NodeEntryInformation entry = elt.getValue();
Messages.NodeEntryInformationProto.Builder node =
Messages.NodeEntryInformationProto.newBuilder();
node.setPriority(entry.priority);
node.setName(elt.getKey());
node.setFailed(entry.failed);
node.setFailedRecently(entry.failedRecently);
node.setLive(entry.live);
node.setLastUsed(entry.lastUsed);
node.setPreempted(entry.preempted);
node.setRequested(entry.requested);
node.setReleasing(entry.releasing);
node.setStartFailed(entry.startFailed);
node.setStarting(entry.starting);
builder.addEntries(node.build());
}
}
return builder.build();
}
public static NodeInformation unmarshall(Messages.NodeInformationProto wire) {
NodeInformation info = new NodeInformation();
info.healthReport = wire.getHealthReport();
info.hostname = wire.getHostname();
info.httpAddress = wire.getHttpAddress();
info.labels = wire.getLabels();
info.lastUpdated = wire.getLastUpdated();
info.rackName = wire.getRackName();
info.state = wire.getState();
List<Messages.NodeEntryInformationProto> entriesList = wire.getEntriesList();
if (entriesList != null) {
info.entries = new HashMap<>(entriesList.size());
for (Messages.NodeEntryInformationProto entry : entriesList) {
NodeEntryInformation nei = new NodeEntryInformation();
nei.failed = entry.getFailed();
nei.failedRecently = entry.getFailedRecently();
nei.lastUsed = entry.getLastUsed();
nei.live = entry.getLive();
nei.preempted = entry.getPreempted();
nei.priority = entry.getPriority();
nei.requested = entry.getRequested();
nei.releasing = entry.getReleasing();
nei.startFailed = entry.getStartFailed();
nei.starting = entry.getStarting();
info.entries.put(entry.getName(), nei);
}
}
return info;
}
public static ContainerInformation unmarshall(Messages.ContainerInformationProto wire) {
ContainerInformation info = new ContainerInformation();
info.containerId = wire.getContainerId();
info.component = wire.getComponent();
info.appVersion = wire.getAppVersion();
info.state = wire.getState();
if (wire.hasReleased()) {
info.released = wire.getReleased();
}
if (wire.hasExitCode()) {
info.exitCode = wire.getExitCode();
}
if (wire.hasDiagnostics()) {
info.diagnostics = wire.getDiagnostics();
}
if (wire.hasHost()) {
info.host = wire.getHost();
}
if (wire.hasHostURL()) {
info.host = wire.getHostURL();
}
info.createTime = wire.getCreateTime();
info.startTime = wire.getStartTime();
info.output = wire.getOutputList().toArray(
new String[wire.getOutputCount()]
);
if (wire.hasPlacement()) {
info.placement = wire.getPlacement();
}
return info;
}
public static List<ContainerInformation> unmarshall(Messages.GetLiveContainersResponseProto wire) {
List<ContainerInformation> infoList = new ArrayList<>(wire.getContainersList().size());
for (Messages.ContainerInformationProto container : wire.getContainersList()) {
infoList.add(unmarshall(container));
}
return infoList;
}
public static Messages.ContainerInformationProto marshall(ContainerInformation info) {
Messages.ContainerInformationProto.Builder builder =
Messages.ContainerInformationProto.newBuilder();
if (info.containerId != null) {
builder.setContainerId(info.containerId);
}
if (info.component != null) {
builder.setComponent(info.component);
}
if (info.appVersion != null) {
builder.setAppVersion(info.appVersion);
}
builder.setCreateTime(info.createTime);
if (info.diagnostics != null) {
builder.setDiagnostics(info.diagnostics);
}
if (info.host != null) {
builder.setHost(info.host);
}
if (info.hostURL != null) {
builder.setHostURL(info.hostURL);
}
if (info.output != null) {
builder.addAllOutput(Arrays.asList(info.output));
}
if (info.released != null) {
builder.setReleased(info.released);
}
if (info.placement != null) {
builder.setPlacement(info.placement);
}
builder.setStartTime(info.startTime);
builder.setState(info.state);
return builder.build();
}
public static String unmarshall(Messages.WrappedJsonProto wire) {
return wire.getJson();
}
public static ConfTree unmarshallToConfTree(Messages.WrappedJsonProto wire) throws
IOException {
return new ConfTreeSerDeser().fromJson(wire.getJson());
}
public static ConfTreeOperations unmarshallToCTO(Messages.WrappedJsonProto wire) throws
IOException {
return new ConfTreeOperations(new ConfTreeSerDeser().fromJson(wire.getJson()));
}
public static AggregateConf unmarshallToAggregateConf(Messages.WrappedJsonProto wire) throws
IOException {
return new AggregateConfSerDeser().fromJson(wire.getJson());
}
}

2293
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java Normal file
View File

File diff suppressed because it is too large Load Diff

53
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api.types;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
/**
* Serialized information about liveness
* <p>
* If true liveness probes are implemented, this
* datatype can be extended to publish their details.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ApplicationLivenessInformation {
/** flag set if the cluster is at size */
public boolean allRequestsSatisfied;
/** number of outstanding requests: those needed to satisfy */
public int requestsOutstanding;
/** number of requests submitted to YARN */
public int activeRequests;
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("ApplicationLivenessInformation{");
sb.append("allRequestsSatisfied=").append(allRequestsSatisfied);
sb.append(", requestsOutstanding=").append(requestsOutstanding);
sb.append('}');
return sb.toString();
}
}

107
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java Normal file
View File

@ -0,0 +1,107 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api.types;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Serializable version of component data.
* <p>
* This is sent in REST calls as a JSON object —but is also marshalled into
* a protobuf structure. Look at {@link org.apache.slider.api.proto.RestTypeMarshalling}
* for the specifics there.
* <p>
* This means that if any fields are added here. they must be added to
* <code>src/main/proto/SliderClusterMessages.proto</code> and
* the protobuf structures rebuilt via a {@code mvn generate-sources -Pcompile-protobuf}
*
* See also {@link RoleStatus#serialize()}
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ComponentInformation {
public String name;
public int priority;
public int desired, actual, releasing;
public int placementPolicy;
public int requested;
public int failed, started, startFailed, completed, totalRequested;
public int nodeFailed, failedRecently, preempted;
public int pendingAntiAffineRequestCount;
public boolean isAARequestOutstanding;
public String failureMessage;
public List<String> containers;
/**
* Build the statistics map from the current data
* @return a map for use in statistics reports
*/
public Map<String, Integer> buildStatistics() {
Map<String, Integer> stats = new HashMap<>();
stats.put(StatusKeys.STATISTICS_CONTAINERS_ACTIVE_REQUESTS, requested);
stats.put(StatusKeys.STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING, pendingAntiAffineRequestCount);
stats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, completed);
stats.put(StatusKeys.STATISTICS_CONTAINERS_DESIRED, desired);
stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED, failed);
stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_NODE, nodeFailed);
stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_RECENTLY, failedRecently);
stats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, actual);
stats.put(StatusKeys.STATISTICS_CONTAINERS_PREEMPTED, preempted);
stats.put(StatusKeys.STATISTICS_CONTAINERS_REQUESTED, totalRequested);
stats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED, started);
stats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, startFailed);
return stats;
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("ComponentInformation{");
sb.append(", name='").append(name).append('\'');
sb.append(", actual=").append(actual);
sb.append(", completed=").append(completed);
sb.append(", desired=").append(desired);
sb.append(", failed=").append(failed);
sb.append(", failureMessage='").append(failureMessage).append('\'');
sb.append(", placementPolicy=").append(placementPolicy);
sb.append(", isAARequestOutstanding=").append(isAARequestOutstanding);
sb.append(", pendingAntiAffineRequestCount=").append(pendingAntiAffineRequestCount);
sb.append(", priority=").append(priority);
sb.append(", releasing=").append(releasing);
sb.append(", requested=").append(requested);
sb.append(", started=").append(started);
sb.append(", startFailed=").append(startFailed);
sb.append(", totalRequested=").append(totalRequested);
sb.append(", container count='")
.append(containers == null ? 0 : containers.size())
.append('\'');
sb.append('}');
return sb.toString();
}
}

58
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api.types;
import org.apache.hadoop.registry.client.binding.JsonSerDeser;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
/**
* Serializable version of component instance data
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ContainerInformation {
public String containerId;
public String component;
public String appVersion;
public Boolean released;
public int state;
public Integer exitCode;
public String diagnostics;
public long createTime;
public long startTime;
public String host;
public String hostURL;
public String placement;
/**
* What is the tail output from the executed process (or [] if not started
* or the log cannot be picked up
*/
public String[] output;
@Override
public String toString() {
JsonSerDeser<ContainerInformation> serDeser =
new JsonSerDeser<>(
ContainerInformation.class);
return serDeser.toString(this);
}
}

78
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java Normal file
View File

@ -0,0 +1,78 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api.types;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
/**
* Serialized node entry information. Must be kept in sync with the protobuf equivalent.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class NodeEntryInformation {
/** incrementing counter of instances that failed */
public int failed;
/** Counter of "failed recently" events. */
public int failedRecently;
/** timestamp of last use */
public long lastUsed;
/** Number of live nodes. */
public int live;
/** incrementing counter of instances that have been pre-empted. */
public int preempted;
/** Priority */
public int priority;
/** instance explicitly requested on this node */
public int requested;
/** number of containers being released off this node */
public int releasing;
/** incrementing counter of instances that failed to start */
public int startFailed;
/** number of starting instances */
public int starting;
@Override
public String toString() {
final StringBuilder sb = new StringBuilder(
"NodeEntryInformation{");
sb.append("priority=").append(priority);
sb.append(", live=").append(live);
sb.append(", requested=").append(requested);
sb.append(", releasing=").append(releasing);
sb.append(", starting=").append(starting);
sb.append(", failed=").append(failed);
sb.append(", failedRecently=").append(failedRecently);
sb.append(", startFailed=").append(startFailed);
sb.append(", preempted=").append(preempted);
sb.append(", lastUsed=").append(lastUsed);
sb.append('}');
return sb.toString();
}
}

59
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api.types;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Serialized node information. Must be kept in sync with the protobuf equivalent.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class NodeInformation {
public String hostname;
public String state;
public String labels;
public String rackName;
public String httpAddress;
public String healthReport;
public long lastUpdated;
public Map<String, NodeEntryInformation> entries = new HashMap<>();
@Override
public String toString() {
final StringBuilder sb = new StringBuilder(
"NodeInformation{");
sb.append("hostname='").append(hostname).append('\'');
sb.append(", state='").append(state).append('\'');
sb.append(", labels='").append(labels).append('\'');
sb.append(", rackName='").append(rackName).append('\'');
sb.append(", httpAddress='").append(httpAddress).append('\'');
sb.append(", healthReport='").append(healthReport).append('\'');
sb.append(", lastUpdated=").append(lastUpdated);
sb.append('}');
return sb.toString();
}
}

41
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api.types;
import org.apache.slider.core.persist.JsonSerDeser;
import java.util.ArrayList;
import java.util.Collection;
public class NodeInformationList extends ArrayList<NodeInformation> {
public NodeInformationList() {
}
public NodeInformationList(Collection<? extends NodeInformation> c) {
super(c);
}
public NodeInformationList(int initialCapacity) {
super(initialCapacity);
}
public static JsonSerDeser<NodeInformationList> createSerializer() {
return new JsonSerDeser<>(NodeInformationList.class);
}
}

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/PingInformation.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api.types;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
/**
* Serialized information to/from Ping operations
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class PingInformation {
public long time;
public String text;
public String verb;
public String body;
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("PingResource{");
sb.append("time=").append(time);
sb.append(", verb=").append(verb);
sb.append(", text='").append(text).append('\'');
sb.append(", body='").append(body).append('\'');
sb.append('}');
return sb.toString();
}
}

66
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api.types;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
/**
* Simple role statistics for state views; can be generated by RoleStatus
* instances, and aggregated for summary information.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class RoleStatistics {
public long activeAA = 0L;
public long actual = 0L;
public long completed = 0L;
public long desired = 0L;
public long failed = 0L;
public long failedRecently = 0L;
public long limitsExceeded = 0L;
public long nodeFailed = 0L;
public long preempted = 0L;
public long releasing = 0L;
public long requested = 0L;
public long started = 0L;
public long startFailed = 0L;
public long totalRequested = 0L;
/**
* Add another statistics instance
* @param that the other value
* @return this entry
*/
public RoleStatistics add(final RoleStatistics that) {
activeAA += that.activeAA;
actual += that.actual;
completed += that.completed;
desired += that.desired;
failed += that.failed;
failedRecently += that.failedRecently;
limitsExceeded += that.limitsExceeded;
nodeFailed += that.nodeFailed;
preempted += that.preempted;
releasing += that.releasing;
requested += that.requested;
started += that.started;
startFailed += that.totalRequested;
totalRequested += that.totalRequested;
return this;
}
}

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.api.types;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
/**
* Description of a slider instance
*/
public class SliderInstanceDescription {
public final String name;
public final Path path;
public final ApplicationReport applicationReport;
public SliderInstanceDescription(String name,
Path path,
ApplicationReport applicationReport) {
this.name = name;
this.path = path;
this.applicationReport = applicationReport;
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("SliderInstanceDescription{");
sb.append("name='").append(name).append('\'');
sb.append(", path=").append(path);
sb.append(", applicationReport: ")
.append(applicationReport == null
? "null"
: (" id " + applicationReport.getApplicationId()));
sb.append('}');
return sb.toString();
}
}

201
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientRegistryBinder.java Normal file
View File

@ -0,0 +1,201 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.client;
import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.encodeForRegistry;
import static org.apache.hadoop.registry.client.binding.RegistryUtils.convertUsername;
import static org.apache.hadoop.registry.client.binding.RegistryUtils.getCurrentUsernameUnencoded;
import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
/**
* Generic code to get the URLs for clients via the registry
*/
public class ClientRegistryBinder {
private static final Logger log =
LoggerFactory.getLogger(ClientRegistryBinder.class);
private final RegistryOperations operations;
public ClientRegistryBinder(RegistryOperations operations) {
this.operations = operations;
}
/**
* Buld the user path -switches to the system path if the user is "".
* It also cross-converts the username to ascii via punycode
* @param username username or ""
* @return the path to the user
*/
public static String homePathForUser(String username) {
Preconditions.checkArgument(username != null, "null user");
// catch recursion
if (username.startsWith(RegistryConstants.PATH_USERS)) {
return username;
}
if (username.isEmpty()) {
return RegistryConstants.PATH_SYSTEM_SERVICES;
}
// convert username to registry name
String convertedName = convertUsername(username);
return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
encodeForRegistry(convertedName));
}
/**
* Get the current username, before any encoding has been applied.
* @return the current user from the kerberos identity, falling back
* to the user and/or env variables.
*/
public static String currentUsernameUnencoded() {
String env_hadoop_username = System.getenv(
RegistryInternalConstants.HADOOP_USER_NAME);
return getCurrentUsernameUnencoded(env_hadoop_username);
}
/**
* Qualify a user.
* <ol>
* <li> <code>"~"</code> maps to user home path home</li>
* <li> <code>"~user"</code> maps to <code>/users/$user</code></li>
* <li> <code>"/"</code> maps to <code>/services/</code></li>
* </ol>
* @param user the username
* @return the base path
*/
public static String qualifyUser(String user) {
// qualify the user
String t = user.trim();
if (t.startsWith("/")) {
// already resolved
return t;
} else if (t.equals("~")) {
// self
return currentUsernameUnencoded();
} else if (t.startsWith("~")) {
// another user
// convert username to registry name
String convertedName = convertUsername(t.substring(1));
return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
encodeForRegistry(convertedName));
} else {
return "/" + t;
}
}
/**
* Look up an external REST API
* @param user user which will be qualified as per {@link #qualifyUser(String)}
* @param serviceClass service class
* @param instance instance name
* @param api API
* @return the API, or an exception is raised.
* @throws IOException
*/
public String lookupExternalRestAPI(String user,
String serviceClass,
String instance,
String api)
throws IOException {
String qualified = qualifyUser(user);
String path = servicePath(qualified, serviceClass, instance);
String restAPI = resolveExternalRestAPI(api, path);
if (restAPI == null) {
throw new PathNotFoundException(path + " API " + api);
}
return restAPI;
}
/**
* Resolve a service record then return an external REST API exported it.
*
* @param api API to resolve
* @param path path of the service record
* @return null if the record exists but the API is absent or it has no
* REST endpoints.
* @throws IOException resolution problems, as covered in
* {@link RegistryOperations#resolve(String)}
*/
protected String resolveExternalRestAPI(String api, String path) throws
IOException {
ServiceRecord record = operations.resolve(path);
return lookupRestAPI(record, api, true);
}
/**
* Look up an external REST API endpoint
* @param record service record
* @param api URI of api
* @param external flag to indicate this is an external record
* @return the first endpoint of the implementation, or null if there
* is no entry for the API, implementation or it's the wrong type.
*/
public static String lookupRestAPI(ServiceRecord record,
String api, boolean external) throws InvalidRecordException {
try {
String url = null;
Endpoint endpoint = getEndpoint(record, api, external);
List<String> addresses =
RegistryTypeUtils.retrieveAddressesUriType(endpoint);
if (addresses != null && !addresses.isEmpty()) {
url = addresses.get(0);
}
return url;
} catch (InvalidRecordException e) {
log.debug("looking for API {}", api, e);
return null;
}
}
/**
* Get an endpont by API
* @param record service record
* @param api API
* @param external flag to indicate this is an external record
* @return the endpoint or null
*/
public static Endpoint getEndpoint(ServiceRecord record,
String api,
boolean external) {
return external ? record.getExternalEndpoint(api)
: record.getInternalEndpoint(api);
}
}

111
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java Normal file
View File

@ -0,0 +1,111 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.client;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.NotFoundException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.registry.docstore.ConfigFormat;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
import org.apache.slider.core.registry.retrieve.RegistryRetriever;
import java.io.File;
import java.io.IOException;
import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser;
import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
public class ClientUtils {
public static ServiceRecord lookupServiceRecord(RegistryOperations rops,
String user, String name) throws IOException, SliderException {
return lookupServiceRecord(rops, user, null, name);
}
public static ServiceRecord lookupServiceRecord(RegistryOperations rops,
String user, String type, String name) throws IOException,
SliderException {
if (StringUtils.isEmpty(user)) {
user = currentUser();
} else {
user = RegistryPathUtils.encodeForRegistry(user);
}
if (StringUtils.isEmpty(type)) {
type = SliderKeys.APP_TYPE;
}
String path = servicePath(user, type, name);
return resolve(rops, path);
}
public static ServiceRecord resolve(RegistryOperations rops, String path)
throws IOException, SliderException {
try {
return rops.resolve(path);
} catch (PathNotFoundException | NoRecordException e) {
throw new NotFoundException(e.getPath().toString(), e);
}
}
public static PublishedConfiguration getConfigFromRegistry(
RegistryOperations rops, Configuration configuration,
String configName, String appName, String user, boolean external)
throws IOException, SliderException {
ServiceRecord instance = lookupServiceRecord(rops, user, appName);
RegistryRetriever retriever = new RegistryRetriever(configuration, instance);
PublishedConfigSet configurations = retriever.getConfigurations(external);
PublishedConfiguration published = retriever.retrieveConfiguration(
configurations, configName, external);
return published;
}
public static String saveOrReturnConfig(PublishedConfiguration published,
String format, File destPath, String fileName)
throws BadCommandArgumentsException, IOException {
ConfigFormat configFormat = ConfigFormat.resolve(format);
if (configFormat == null) {
throw new BadCommandArgumentsException(
"Unknown/Unsupported format %s ", format);
}
PublishedConfigurationOutputter outputter =
PublishedConfigurationOutputter.createOutputter(configFormat,
published);
boolean print = destPath == null;
if (!print) {
if (destPath.isDirectory()) {
// creating it under a directory
destPath = new File(destPath, fileName);
}
outputter.save(destPath);
return null;
} else {
return outputter.asString();
}
}
}

4569
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java Normal file
View File

File diff suppressed because it is too large Load Diff

368
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java Normal file
View File

@ -0,0 +1,368 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.client;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.SliderInstanceDescription;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionClientArgs;
import org.apache.slider.common.params.ActionDependencyArgs;
import org.apache.slider.common.params.ActionDestroyArgs;
import org.apache.slider.common.params.ActionDiagnosticArgs;
import org.apache.slider.common.params.ActionEchoArgs;
import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.ActionInstallKeytabArgs;
import org.apache.slider.common.params.ActionInstallPackageArgs;
import org.apache.slider.common.params.ActionKeytabArgs;
import org.apache.slider.common.params.ActionNodesArgs;
import org.apache.slider.common.params.ActionPackageArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionResourceArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.ActionUpgradeArgs;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.providers.AbstractClientProvider;
import java.io.IOException;
import java.util.Map;
/**
* Interface of those method calls in the slider API that are intended
* for direct public invocation.
* <p>
* Stability: evolving
*/
public interface SliderClientAPI extends Service {
/**
* Destroy a cluster. There's two race conditions here
* #1 the cluster is started between verifying that there are no live
* clusters of that name.
*/
int actionDestroy(String clustername, ActionDestroyArgs destroyArgs)
throws YarnException, IOException;
int actionDestroy(String clustername) throws YarnException,
IOException;
/**
* AM to commit an asynchronous suicide
*/
int actionAmSuicide(String clustername,
ActionAMSuicideArgs args) throws YarnException, IOException;
/**
* Get the provider for this cluster
* @param provider the name of the provider
* @return the provider instance
* @throws SliderException problems building the provider
*/
AbstractClientProvider createClientProvider(String provider)
throws SliderException;
/**
* Build up the cluster specification/directory
*
* @param clustername cluster name
* @param buildInfo the arguments needed to build the cluster
* @throws YarnException Yarn problems
* @throws IOException other problems
* @throws BadCommandArgumentsException bad arguments.
*/
int actionBuild(String clustername,
AbstractClusterBuildingActionArgs buildInfo) throws YarnException, IOException;
/**
* Upload keytab to a designated sub-directory of the user home directory
*
* @param installKeytabInfo the arguments needed to upload the keytab
* @throws YarnException Yarn problems
* @throws IOException other problems
* @throws BadCommandArgumentsException bad arguments.
* @deprecated use #actionKeytab
*/
int actionInstallKeytab(ActionInstallKeytabArgs installKeytabInfo)
throws YarnException, IOException;
/**
* Manage keytabs leveraged by slider
*
* @param keytabInfo the arguments needed to manage the keytab
* @throws YarnException Yarn problems
* @throws IOException other problems
* @throws BadCommandArgumentsException bad arguments.
*/
int actionKeytab(ActionKeytabArgs keytabInfo)
throws YarnException, IOException;
/**
* Upload application package to user home directory
*
* @param installPkgInfo the arguments needed to upload the package
* @throws YarnException Yarn problems
* @throws IOException other problems
* @throws BadCommandArgumentsException bad arguments.
*/
int actionInstallPkg(ActionInstallPackageArgs installPkgInfo)
throws YarnException, IOException;
/**
* Manage file resources leveraged by slider
*
* @param resourceInfo the arguments needed to manage the resource
* @throws YarnException Yarn problems
* @throws IOException other problems
* @throws BadCommandArgumentsException bad arguments.
*/
int actionResource(ActionResourceArgs resourceInfo)
throws YarnException, IOException;
/**
* Perform client operations such as install or configure
*
* @param clientInfo the arguments needed for client operations
*
* @throws SliderException bad arguments.
* @throws IOException problems related to package and destination folders
*/
int actionClient(ActionClientArgs clientInfo)
throws IOException, YarnException;
/**
* Managing slider application package
*
* @param pkgInfo the arguments needed to upload, delete or list the package
* @throws YarnException Yarn problems
* @throws IOException other problems
* @throws BadCommandArgumentsException bad arguments.
*/
int actionPackage(ActionPackageArgs pkgInfo)
throws YarnException, IOException;
/**
* Update the cluster specification
*
* @param clustername cluster name
* @param buildInfo the arguments needed to update the cluster
* @throws YarnException Yarn problems
* @throws IOException other problems
*/
int actionUpdate(String clustername,
AbstractClusterBuildingActionArgs buildInfo)
throws YarnException, IOException;
/**
* Upgrade the cluster with a newer version of the application
*
* @param clustername cluster name
* @param buildInfo the arguments needed to upgrade the cluster
* @throws YarnException Yarn problems
* @throws IOException other problems
*/
int actionUpgrade(String clustername,
ActionUpgradeArgs buildInfo)
throws YarnException, IOException;
/**
* Get the report of a this application
* @return the app report or null if it could not be found.
* @throws IOException
* @throws YarnException
*/
ApplicationReport getApplicationReport()
throws IOException, YarnException;
/**
* Kill the submitted application via YARN
* @throws YarnException
* @throws IOException
*/
boolean forceKillApplication(String reason)
throws YarnException, IOException;
/**
* Implement the list action: list all nodes
* @return exit code of 0 if a list was created
*/
int actionList(String clustername, ActionListArgs args) throws IOException, YarnException;
/**
* Enumerate slider instances for the current user, and the
* most recent app report, where available.
* @param listOnlyInState boolean to indicate that the instances should
* only include those in a YARN state
* <code> minAppState &lt;= currentState &lt;= maxAppState </code>
*
* @param minAppState minimum application state to include in enumeration.
* @param maxAppState maximum application state to include
* @return a map of application instance name to description
* @throws IOException Any IO problem
* @throws YarnException YARN problems
*/
Map<String, SliderInstanceDescription> enumSliderInstances(
boolean listOnlyInState,
YarnApplicationState minAppState,
YarnApplicationState maxAppState)
throws IOException, YarnException;
/**
* Implement the islive action: probe for a cluster of the given name existing
* @return exit code
*/
int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException;
/**
* Test for a cluster existing probe for a cluster of the given name existing
* in the filesystem. If the live param is set, it must be a live cluster
* @return exit code
*/
int actionExists(String name, boolean checkLive) throws YarnException, IOException;
/**
* Kill a specific container of the cluster
* @param name cluster name
* @param args arguments
* @return exit code
* @throws YarnException
* @throws IOException
*/
int actionKillContainer(String name, ActionKillContainerArgs args)
throws YarnException, IOException;
/**
* Echo operation (not currently wired up to command line)
* @param name cluster name
* @param args arguments
* @return the echoed text
* @throws YarnException
* @throws IOException
*/
String actionEcho(String name, ActionEchoArgs args)
throws YarnException, IOException;
/**
* Status operation
*
* @param clustername cluster name
* @param statusArgs status arguments
* @return 0 -for success, else an exception is thrown
* @throws YarnException
* @throws IOException
*/
int actionStatus(String clustername, ActionStatusArgs statusArgs)
throws YarnException, IOException;
/**
* Version Details
* @return exit code
*/
int actionVersion();
/**
* Stop the cluster
*
* @param clustername cluster name
* @param freezeArgs arguments to the stop
* @return EXIT_SUCCESS if the cluster was not running by the end of the operation
*/
int actionFreeze(String clustername, ActionFreezeArgs freezeArgs)
throws YarnException, IOException;
/**
* Restore a cluster
*/
int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException;
/**
* Registry operation
*
* @param args registry Arguments
* @return 0 for success, -1 for some issues that aren't errors, just failures
* to retrieve information (e.g. no configurations for that entry)
* @throws YarnException YARN problems
* @throws IOException Network or other problems
*/
int actionResolve(ActionResolveArgs args)
throws YarnException, IOException;
/**
* Registry operation
*
* @param registryArgs registry Arguments
* @return 0 for success, -1 for some issues that aren't errors, just failures
* to retrieve information (e.g. no configurations for that entry)
* @throws YarnException YARN problems
* @throws IOException Network or other problems
*/
int actionRegistry(ActionRegistryArgs registryArgs)
throws YarnException, IOException;
/**
* diagnostic operation
*
* @param diagnosticArgs diagnostic Arguments
* @return 0 for success, -1 for some issues that aren't errors, just
* failures to retrieve information (e.g. no application name
* specified)
* @throws YarnException YARN problems
* @throws IOException Network or other problems
*/
int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs);
/**
* Get the registry binding. As this may start the registry, it can take time
* and fail
* @return the registry
*/
RegistryOperations getRegistryOperations()
throws SliderException, IOException;
/**
* Upload all Slider AM and agent dependency libraries to HDFS, so that they
* do not need to be uploaded with every create call. This operation is
* Slider version specific. So it needs to be invoked for every single
* version of slider/slider-client.
*
* @throws SliderException
* @throws IOException
*/
int actionDependency(ActionDependencyArgs dependencyArgs) throws IOException,
YarnException;
/**
* List the nodes
* @param args
* @return
* @throws YarnException
* @throws IOException
*/
NodeInformationList listYarnClusterNodes(ActionNodesArgs args)
throws YarnException, IOException;
}

410
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java Normal file
View File

@ -0,0 +1,410 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.client;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.slider.api.types.NodeInformation;
import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.params.ActionNodesArgs;
import org.apache.slider.common.tools.CoreFileSystem;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.BindException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* A class that extends visibility to some of the YarnClientImpl
* members and data structures, and factors out pure-YARN operations
* from the slider entry point service
*/
public class SliderYarnClientImpl extends YarnClientImpl {
protected static final Logger log = LoggerFactory.getLogger(SliderYarnClientImpl.class);
/**
* Keyword to use in the {@link #emergencyForceKill(String)}
* operation to force kill <i>all</i> application instances belonging
* to a specific user
*/
public static final String KILL_ALL = "all";
@Override
protected void serviceInit(Configuration conf) throws Exception {
InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(conf);
if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
// address isn't known; fail fast
throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS
+ " value:" + conf.get(YarnConfiguration.RM_ADDRESS)
+ " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort");
}
super.serviceInit(conf);
}
/**
* Get the RM Client RPC interface
* @return an RPC interface valid after initialization and authentication
*/
public ApplicationClientProtocol getRmClient() {
return rmClient;
}
/**
* List Slider <i>running</i>instances belonging to a specific user.
* @deprecated use {@link #listDeployedInstances(String)}
* @param user user: "" means all users
* @return a possibly empty list of Slider AMs
*/
public List<ApplicationReport> listInstances(String user)
throws YarnException, IOException {
return listDeployedInstances(user);
}
/**
* List Slider <i>deployed</i>instances belonging to a specific user.
* <p>
* Deployed means: known about in the YARN cluster; it will include
* any that are in the failed/finished state, as well as those queued
* for starting.
* @param user user: "" means all users
* @return a possibly empty list of Slider AMs
*/
public List<ApplicationReport> listDeployedInstances(String user)
throws YarnException, IOException {
Preconditions.checkArgument(user != null, "Null User");
Set<String> types = new HashSet<>(1);
types.add(SliderKeys.APP_TYPE);
List<ApplicationReport> allApps = getApplications(types);
List<ApplicationReport> results = new ArrayList<>();
for (ApplicationReport report : allApps) {
if (StringUtils.isEmpty(user) || user.equals(report.getUser())) {
results.add(report);
}
}
return results;
}
/**
* find all instances of a specific app -if there is more than one in the
* YARN cluster,
* this returns them all
* @param user user; use "" for all users
* @param appname application name
* @return the list of all matching application instances
*/
public List<ApplicationReport> findAllInstances(String user,
String appname)
throws IOException, YarnException {
Preconditions.checkArgument(appname != null, "Null application name");
List<ApplicationReport> instances = listDeployedInstances(user);
List<ApplicationReport> results =
new ArrayList<>(instances.size());
for (ApplicationReport report : instances) {
if (report.getName().equals(appname)) {
results.add(report);
}
}
return results;
}
/**
* Helper method to determine if a cluster application is running -or
* is earlier in the lifecycle
* @param app application report
* @return true if the application is considered live
*/
public boolean isApplicationLive(ApplicationReport app) {
Preconditions.checkArgument(app != null, "Null app report");
return app.getYarnApplicationState().ordinal() <= YarnApplicationState.RUNNING.ordinal();
}
/**
* Kill a running application
* @param applicationId app Id
* @param reason reason: reason for log
* @return the response
* @throws YarnException YARN problems
* @throws IOException IO problems
*/
public KillApplicationResponse killRunningApplication(ApplicationId applicationId,
String reason)
throws YarnException, IOException {
Preconditions.checkArgument(applicationId != null, "Null application Id");
log.info("Killing application {} - {}", applicationId.getClusterTimestamp(),
reason);
KillApplicationRequest request =
Records.newRecord(KillApplicationRequest.class);
request.setApplicationId(applicationId);
return getRmClient().forceKillApplication(request);
}
private String getUsername() throws IOException {
return UserGroupInformation.getCurrentUser().getShortUserName();
}
/**
* Force kill a yarn application by ID. No niceties here
* @param applicationId app Id. "all" means "kill all instances of the current user
*
*/
public void emergencyForceKill(String applicationId)
throws YarnException, IOException {
Preconditions.checkArgument(StringUtils.isNotEmpty(applicationId),
"Null/empty application Id");
if (KILL_ALL.equals(applicationId)) {
// user wants all instances killed
String user = getUsername();
log.info("Killing all applications belonging to {}", user);
Collection<ApplicationReport> instances = listDeployedInstances(user);
for (ApplicationReport instance : instances) {
if (isApplicationLive(instance)) {
ApplicationId appId = instance.getApplicationId();
log.info("Killing Application {}", appId);
killRunningApplication(appId, "forced kill");
}
}
} else {
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
log.info("Killing Application {}", applicationId);
killRunningApplication(appId, "forced kill");
}
}
/**
* Monitor the submitted application for reaching the requested state.
* Will also report if the app reaches a later state (failed, killed, etc)
* Kill application if duration!= null & time expires.
* @param appId Application Id of application to be monitored
* @param duration how long to wait -must be more than 0
* @param desiredState desired state.
* @return the application report -null on a timeout
* @throws YarnException
* @throws IOException
*/
public ApplicationReport monitorAppToState(
ApplicationId appId, YarnApplicationState desiredState, Duration duration)
throws YarnException, IOException {
if (appId == null) {
throw new BadCommandArgumentsException("null application ID");
}
if (duration.limit <= 0) {
throw new BadCommandArgumentsException("Invalid monitoring duration");
}
log.debug("Waiting {} millis for app to reach state {} ",
duration.limit,
desiredState);
duration.start();
try {
while (true) {
// Get application report for the appId we are interested in
ApplicationReport r = getApplicationReport(appId);
log.debug("queried status is\n{}",
new SliderUtils.OnDemandReportStringifier(r));
YarnApplicationState state = r.getYarnApplicationState();
if (state.ordinal() >= desiredState.ordinal()) {
log.debug("App in desired state (or higher) :{}", state);
return r;
}
if (duration.getLimitExceeded()) {
log.debug(
"Wait limit of {} millis to get to state {}, exceeded; app status\n {}",
duration.limit,
desiredState,
new SliderUtils.OnDemandReportStringifier(r));
return null;
}
// sleep 1s.
try {
Thread.sleep(1000);
} catch (InterruptedException ignored) {
log.debug("Thread sleep in monitoring loop interrupted");
}
}
} finally {
duration.close();
}
}
/**
* find all live instances of a specific app -if there is >1 in the cluster,
* this returns them all. State should be running or less
* @param user user
* @param appname application name
* @return the list of all matching application instances
*/
public List<ApplicationReport> findAllLiveInstances(String user,
String appname) throws
YarnException,
IOException {
Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
"Null/empty application name");
List<ApplicationReport> instances = listDeployedInstances(user);
List<ApplicationReport> results =
new ArrayList<ApplicationReport>(instances.size());
for (ApplicationReport app : instances) {
if (app.getName().equals(appname)
&& isApplicationLive(app)) {
results.add(app);
}
}
return results;
}
/**
* Find a cluster in the instance list; biased towards live instances
* @param instances list of instances
* @param appname application name
* @return the first found instance, else a failed/finished instance, or null
* if there are none of those
*/
public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
String appname) {
Preconditions.checkArgument(instances != null, "Null instances list");
Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
"Null/empty application name");
// sort by most recent
SliderUtils.sortApplicationsByMostRecent(instances);
ApplicationReport found = null;
for (ApplicationReport app : instances) {
if (app.getName().equals(appname)) {
if (isApplicationLive(app)) {
return app;
}
// set the found value if not set
found = found != null ? found : app;
}
}
return found;
}
/**
* Find an app in the instance list in the desired state
* @param instances instance list
* @param appname application name
* @param desiredState yarn state desired
* @return the match or null for none
*/
public ApplicationReport findAppInInstanceList(List<ApplicationReport> instances,
String appname,
YarnApplicationState desiredState) {
Preconditions.checkArgument(instances != null, "Null instances list");
Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
"Null/empty application name");
Preconditions.checkArgument(desiredState != null, "Null desiredState");
log.debug("Searching {} records for instance name {} in state '{}'",
instances.size(), appname, desiredState);
for (ApplicationReport app : instances) {
if (app.getName().equals(appname)) {
YarnApplicationState appstate =
app.getYarnApplicationState();
log.debug("app ID {} is in state {}", app.getApplicationId(), appstate);
if (appstate.equals(desiredState)) {
log.debug("match");
return app;
}
}
}
// nothing found in desired state
log.debug("No match");
return null;
}
/**
* List the nodes in the cluster, possibly filtering by node state or label.
*
* @param label label to filter by -or "" for any
* @param live flag to request running nodes only
* @return a possibly empty list of nodes in the cluster
* @throws IOException IO problems
* @throws YarnException YARN problems
*/
public NodeInformationList listNodes(String label, boolean live)
throws IOException, YarnException {
Preconditions.checkArgument(label != null, "null label");
NodeState[] states;
if (live) {
states = new NodeState[1];
states[0] = NodeState.RUNNING;
} else {
states = new NodeState[0];
}
List<NodeReport> reports = getNodeReports(states);
NodeInformationList results = new NodeInformationList(reports.size());
for (NodeReport report : reports) {
if (live && report.getNodeState() != NodeState.RUNNING) {
continue;
}
if (!label.isEmpty() && !report.getNodeLabels().contains(label)) {
continue;
}
// build node info from report
NodeInformation info = new NodeInformation();
info.hostname = report.getNodeId().getHost();
info.healthReport = report.getHealthReport();
info.httpAddress = report.getHttpAddress();
info.labels = SliderUtils.extractNodeLabel(report);
info.rackName = report.getRackName();
info.state = report.getNodeState().toString();
results.add(info);
}
return results;
}
}

109
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java Normal file
View File

@ -0,0 +1,109 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.client;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.common.params.ActionTokensArgs;
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.NotFoundException;
import static org.apache.slider.core.launch.CredentialUtils.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
public class TokensOperation {
private static final Logger log = LoggerFactory.getLogger(TokensOperation.class);
public static final String E_INSECURE
= "Cluster is not secure -tokens cannot be acquired";
public static final String E_MISSING_SOURCE_FILE = "Missing source file: ";
public static final String E_NO_KEYTAB = "No keytab: ";
public int actionTokens(ActionTokensArgs args, FileSystem fs,
Configuration conf,
YarnClientImpl yarnClient)
throws IOException, YarnException {
Credentials credentials;
String footnote = "";
UserGroupInformation user = UserGroupInformation.getCurrentUser();
boolean isSecure = UserGroupInformation.isSecurityEnabled();
if (args.keytab != null) {
File keytab = args.keytab;
if (!keytab.isFile()) {
throw new NotFoundException(E_NO_KEYTAB + keytab.getAbsolutePath());
}
String principal = args.principal;
log.info("Logging in as {} from keytab {}", principal, keytab);
user = UserGroupInformation.loginUserFromKeytabAndReturnUGI(
principal, keytab.getCanonicalPath());
}
Credentials userCredentials = user.getCredentials();
File output = args.output;
if (output != null) {
if (!isSecure) {
throw new BadClusterStateException(E_INSECURE);
}
credentials = new Credentials(userCredentials);
// filesystem
addRMRenewableFSDelegationTokens(conf, fs, credentials);
addRMDelegationToken(yarnClient, credentials);
if (maybeAddTimelineToken(conf, credentials) != null) {
log.debug("Added timeline token");
}
saveTokens(output, credentials);
String filename = output.getCanonicalPath();
footnote = String.format("%d tokens saved to %s\n" +
"To use these in the environment:\n" +
"export %s=%s",
credentials.numberOfTokens(),
filename, UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, filename);
} else if (args.source != null) {
File source = args.source;
log.info("Reading credentials from file {}", source);
if (!source.isFile()) {
throw new NotFoundException( E_MISSING_SOURCE_FILE + source.getAbsolutePath());
}
credentials = Credentials.readTokenStorageFile(args.source, conf);
} else {
StringBuffer origin = new StringBuffer();
File file = locateEnvCredentials(System.getenv(), conf,
origin);
if (file != null) {
log.info("Credential Source {}", origin);
} else {
log.info("Credential source: logged in user");
}
credentials = userCredentials;
}
// list the tokens
log.info("\n{}", dumpTokens(credentials, "\n"));
if (!footnote.isEmpty()) {
log.info(footnote);
}
return 0;
}
}

245
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java Normal file
View File

@ -0,0 +1,245 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.client.ipc;
import com.google.common.base.Preconditions;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.types.NodeInformation;
import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.PingInformation;
import org.apache.slider.api.SliderApplicationApi;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Map;
/**
* Implementation of the Slider RESTy Application API over IPC.
* <p>
* Operations are executed via the {@link SliderClusterOperations}
* instance passed in; raised exceptions may be converted into ones
* consistent with the REST API.
*/
public class SliderApplicationIpcClient implements SliderApplicationApi {
private static final Logger log =
LoggerFactory.getLogger(SliderApplicationIpcClient.class);
private final SliderClusterOperations operations;
public SliderApplicationIpcClient(SliderClusterOperations operations) {
Preconditions.checkArgument(operations != null, "null operations");
this.operations = operations;
}
/**
* Convert received (And potentially unmarshalled) local/remote
* exceptions into the equivalents in the REST API.
* Best effort.
* <p>
* If there is no translation, the original exception is returned.
* <p>
* If a new exception was created, it will have the message of the
* string value of the original exception, and that original
* exception will be the nested cause of this one
* @param exception IOException to convert
* @return an exception to throw
*/
private IOException convert(IOException exception) {
IOException result = exception;
if (exception instanceof NoSuchNodeException) {
result = new FileNotFoundException(exception.toString());
result.initCause(exception);
} else {
// TODO: remap any other exceptions
}
return result;
}
public SliderApplicationIpcClient(SliderClusterProtocol proxy) {
this(new SliderClusterOperations(proxy));
}
@Override
public AggregateConf getDesiredModel() throws IOException {
try {
return operations.getModelDesired();
} catch (IOException e) {
throw convert(e);
}
}
@Override
public ConfTreeOperations getDesiredAppconf() throws IOException {
try {
return operations.getModelDesiredAppconf();
} catch (IOException e) {
throw convert(e);
}
}
@Override
public ConfTreeOperations getDesiredResources() throws IOException {
try {
return operations.getModelDesiredResources();
} catch (IOException e) {
throw convert(e);
}
}
@Override
public void putDesiredResources(ConfTree updated) throws IOException {
try {
operations.flex(updated);
} catch (IOException e) {
throw convert(e);
}
}
@Override
public AggregateConf getResolvedModel() throws IOException {
try {
return operations.getModelResolved();
} catch (IOException e) {
throw convert(e);
}
}
@Override
public ConfTreeOperations getResolvedAppconf() throws IOException {
try {
return operations.getModelResolvedAppconf();
} catch (IOException e) {
throw convert(e);
}
}
@Override
public ConfTreeOperations getResolvedResources() throws IOException {
try {
return operations.getModelResolvedResources();
} catch (IOException e) {
throw convert(e);
}
}
@Override
public ConfTreeOperations getLiveResources() throws IOException {
try {
return operations.getLiveResources();
} catch (IOException e) {
throw convert(e);
}
}
@Override
public Map<String, ContainerInformation> enumContainers() throws IOException {
try {
return operations.enumContainers();
} catch (IOException e) {
throw convert(e);
}
}
@Override
public ContainerInformation getContainer(String containerId) throws
IOException {
try {
return operations.getContainer(containerId);
} catch (IOException e) {
throw convert(e);
}
}
@Override
public Map<String, ComponentInformation> enumComponents() throws IOException {
try {
return operations.enumComponents();
} catch (IOException e) {
throw convert(e);
}
}
@Override
public ComponentInformation getComponent(String componentName) throws IOException {
try {
return operations.getComponent(componentName);
} catch (IOException e) {
throw convert(e);
}
}
@Override
public NodeInformationList getLiveNodes() throws IOException {
try {
return operations.getLiveNodes();
} catch (IOException e) {
throw convert(e);
}
}
@Override
public NodeInformation getLiveNode(String hostname) throws IOException {
try {
return operations.getLiveNode(hostname);
} catch (IOException e) {
throw convert(e);
}
}
@Override
public PingInformation ping(String text) throws IOException {
return null;
}
@Override
public void stop(String text) throws IOException {
try {
operations.stop(text);
} catch (IOException e) {
throw convert(e);
}
}
@Override
public ApplicationLivenessInformation getApplicationLiveness() throws
IOException {
try {
return operations.getApplicationLiveness();
} catch (IOException e) {
throw convert(e);
}
}
@Override
public String toString() {
return "IPC implementation of SliderApplicationApi bonded to " + operations;
}
}

529
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java Normal file
View File

@ -0,0 +1,529 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.client.ipc;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.StateValues;
import org.apache.slider.api.proto.Messages;
import static org.apache.slider.api.proto.RestTypeMarshalling.*;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.types.NodeInformation;
import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.PingInformation;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.WaitTimeoutException;
import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.apache.slider.server.services.security.SecurityStore;
import org.apache.slider.server.services.security.SignCertResponse;
import org.codehaus.jackson.JsonParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Cluster operations at a slightly higher level than the RPC code
*/
public class SliderClusterOperations {
protected static final Logger
log = LoggerFactory.getLogger(SliderClusterOperations.class);
private final SliderClusterProtocol appMaster;
private static final Messages.EmptyPayloadProto EMPTY;
static {
EMPTY = Messages.EmptyPayloadProto.newBuilder().build();
}
public SliderClusterOperations(SliderClusterProtocol appMaster) {
this.appMaster = appMaster;
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("SliderClusterOperations{");
sb.append("IPC binding=").append(appMaster);
sb.append('}');
return sb.toString();
}
/**
* Get a node from the AM
* @param uuid uuid of node
* @return deserialized node
* @throws IOException IO problems
* @throws NoSuchNodeException if the node isn't found
*/
public ClusterNode getNode(String uuid)
throws IOException, NoSuchNodeException, YarnException {
Messages.GetNodeRequestProto req =
Messages.GetNodeRequestProto.newBuilder().setUuid(uuid).build();
Messages.GetNodeResponseProto node = appMaster.getNode(req);
return ClusterNode.fromProtobuf(node.getClusterNode());
}
/**
* Unmarshall a list of nodes from a protobud response
* @param nodes node list
* @return possibly empty list of cluster nodes
* @throws IOException
*/
public List<ClusterNode> convertNodeWireToClusterNodes(List<Messages.RoleInstanceState> nodes)
throws IOException {
List<ClusterNode> nodeList = new ArrayList<>(nodes.size());
for (Messages.RoleInstanceState node : nodes) {
nodeList.add(ClusterNode.fromProtobuf(node));
}
return nodeList;
}
/**
* Echo text (debug action)
* @param text text
* @return the text, echoed back
* @throws YarnException
* @throws IOException
*/
public String echo(String text) throws YarnException, IOException {
Messages.EchoRequestProto.Builder builder =
Messages.EchoRequestProto.newBuilder();
builder.setText(text);
Messages.EchoRequestProto req = builder.build();
Messages.EchoResponseProto response = appMaster.echo(req);
return response.getText();
}
/**
* Connect to a live cluster and get its current state
* @return its description
*/
public ClusterDescription getClusterDescription()
throws YarnException, IOException {
Messages.GetJSONClusterStatusRequestProto req =
Messages.GetJSONClusterStatusRequestProto.newBuilder().build();
Messages.GetJSONClusterStatusResponseProto resp =
appMaster.getJSONClusterStatus(req);
String statusJson = resp.getClusterSpec();
try {
return ClusterDescription.fromJson(statusJson);
} catch (JsonParseException e) {
log.error("Exception " + e + " parsing:\n" + statusJson, e);
throw e;
}
}
/**
* Get the AM instance definition.
* <p>
* See {@link SliderClusterProtocol#getInstanceDefinition(Messages.GetInstanceDefinitionRequestProto)}
* @return current slider AM aggregate definition
* @throws YarnException
* @throws IOException
*/
public AggregateConf getInstanceDefinition()
throws YarnException, IOException {
Messages.GetInstanceDefinitionRequestProto.Builder builder =
Messages.GetInstanceDefinitionRequestProto.newBuilder();
Messages.GetInstanceDefinitionRequestProto request = builder.build();
Messages.GetInstanceDefinitionResponseProto response =
appMaster.getInstanceDefinition(request);
ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
ConfTree internal = confTreeSerDeser.fromJson(response.getInternal());
ConfTree resources = confTreeSerDeser.fromJson(response.getResources());
ConfTree app = confTreeSerDeser.fromJson(response.getApplication());
AggregateConf instanceDefinition =
new AggregateConf(resources, app, internal);
return instanceDefinition;
}
/**
* Kill a container
* @param id container ID
* @return a success flag
* @throws YarnException
* @throws IOException
*/
public boolean killContainer(String id) throws
YarnException,
IOException {
Messages.KillContainerRequestProto.Builder builder =
Messages.KillContainerRequestProto.newBuilder();
builder.setId(id);
Messages.KillContainerRequestProto req = builder.build();
Messages.KillContainerResponseProto response = appMaster.killContainer(req);
return response.getSuccess();
}
/**
* List all node UUIDs in a role
* @param role role name or "" for all
* @return an array of UUID strings
* @throws IOException
* @throws YarnException
*/
public String[] listNodeUUIDsByRole(String role) throws IOException, YarnException {
Collection<String> uuidList = innerListNodeUUIDSByRole(role);
String[] uuids = new String[uuidList.size()];
return uuidList.toArray(uuids);
}
public List<String> innerListNodeUUIDSByRole(String role) throws IOException, YarnException {
Messages.ListNodeUUIDsByRoleRequestProto req =
Messages.ListNodeUUIDsByRoleRequestProto
.newBuilder()
.setRole(role)
.build();
Messages.ListNodeUUIDsByRoleResponseProto resp = appMaster.listNodeUUIDsByRole(req);
return resp.getUuidList();
}
/**
* List all nodes in a role. This is a double round trip: once to list
* the nodes in a role, another to get their details
* @param role
* @return an array of ContainerNode instances
* @throws IOException
* @throws YarnException
*/
public List<ClusterNode> listClusterNodesInRole(String role)
throws IOException, YarnException {
Collection<String> uuidList = innerListNodeUUIDSByRole(role);
Messages.GetClusterNodesRequestProto req =
Messages.GetClusterNodesRequestProto
.newBuilder()
.addAllUuid(uuidList)
.build();
Messages.GetClusterNodesResponseProto resp = appMaster.getClusterNodes(req);
return convertNodeWireToClusterNodes(resp.getClusterNodeList());
}
/**
* Get the details on a list of uuids
* @param uuids instance IDs
* @return a possibly empty list of node details
* @throws IOException
* @throws YarnException
*/
@VisibleForTesting
public List<ClusterNode> listClusterNodes(String[] uuids)
throws IOException, YarnException {
Messages.GetClusterNodesRequestProto req =
Messages.GetClusterNodesRequestProto
.newBuilder()
.addAllUuid(Arrays.asList(uuids))
.build();
Messages.GetClusterNodesResponseProto resp = appMaster.getClusterNodes(req);
return convertNodeWireToClusterNodes(resp.getClusterNodeList());
}
/**
* Wait for an instance of a named role to be live (or past it in the lifecycle)
* @param role role to look for
* @param timeout time to wait
* @return the state. If still in CREATED, the cluster didn't come up
* in the time period. If LIVE, all is well. If >LIVE, it has shut for a reason
* @throws IOException IO
* @throws SliderException Slider
* @throws WaitTimeoutException if the wait timed out
*/
@VisibleForTesting
public int waitForRoleInstanceLive(String role, long timeout)
throws WaitTimeoutException, IOException, YarnException {
Duration duration = new Duration(timeout);
duration.start();
boolean live = false;
int state = StateValues.STATE_CREATED;
log.info("Waiting {} millis for a live node in role {}", timeout, role);
try {
while (!live) {
// see if there is a node in that role yet
List<String> uuids = innerListNodeUUIDSByRole(role);
String[] containers = uuids.toArray(new String[uuids.size()]);
int roleCount = containers.length;
ClusterNode roleInstance = null;
if (roleCount != 0) {
// if there is, get the node
roleInstance = getNode(containers[0]);
if (roleInstance != null) {
state = roleInstance.state;
live = state >= StateValues.STATE_LIVE;
}
}
if (!live) {
if (duration.getLimitExceeded()) {
throw new WaitTimeoutException(
String.format("Timeout after %d millis" +
" waiting for a live instance of type %s; " +
"instances found %d %s",
timeout, role, roleCount,
(roleInstance != null
? (" instance -\n" + roleInstance.toString())
: "")
));
} else {
try {
Thread.sleep(1000);
} catch (InterruptedException ignored) {
// ignored
}
}
}
}
} finally {
duration.close();
}
return state;
}
/**
* Flex operation
* @param resources new resources
* @return the response
* @throws IOException
*/
public boolean flex(ConfTree resources) throws IOException {
Messages.FlexClusterRequestProto request =
Messages.FlexClusterRequestProto.newBuilder()
.setClusterSpec(resources.toJson())
.build();
Messages.FlexClusterResponseProto response = appMaster.flexCluster(request);
return response.getResponse();
}
/**
* Commit (possibly delayed) AM suicide
*
* @param signal exit code
* @param text text text to log
* @param delay delay in millis
* @throws YarnException
* @throws IOException
*/
public void amSuicide(String text, int signal, int delay)
throws IOException {
Messages.AMSuicideRequestProto.Builder builder =
Messages.AMSuicideRequestProto.newBuilder();
if (text != null) {
builder.setText(text);
}
builder.setSignal(signal);
builder.setDelay(delay);
Messages.AMSuicideRequestProto req = builder.build();
appMaster.amSuicide(req);
}
/**
* Get the application liveness
* @return current liveness information
* @throws IOException
*/
public ApplicationLivenessInformation getLivenessInformation() throws IOException {
Messages.GetApplicationLivenessRequestProto.Builder builder =
Messages.GetApplicationLivenessRequestProto.newBuilder();
Messages.ApplicationLivenessInformationProto wire =
appMaster.getLivenessInformation(builder.build());
return unmarshall(wire);
}
public AggregateConf getModelDesired() throws IOException {
return unmarshallToAggregateConf(appMaster.getModelDesired(EMPTY));
}
public ConfTreeOperations getModelDesiredAppconf() throws IOException {
return unmarshallToCTO(appMaster.getModelDesiredAppconf(EMPTY));
}
public ConfTreeOperations getModelDesiredResources() throws IOException {
return unmarshallToCTO(appMaster.getModelDesiredResources(EMPTY));
}
public AggregateConf getModelResolved() throws IOException {
return unmarshallToAggregateConf(appMaster.getModelResolved(EMPTY));
}
public ConfTreeOperations getModelResolvedAppconf() throws IOException {
return unmarshallToCTO(appMaster.getModelResolvedAppconf(EMPTY));
}
public ConfTreeOperations getModelResolvedResources() throws IOException {
return unmarshallToCTO(appMaster.getModelDesiredResources(EMPTY));
}
public ConfTreeOperations getLiveResources() throws IOException {
return unmarshallToCTO(appMaster.getLiveResources(EMPTY));
}
public Map<String, ContainerInformation> enumContainers() throws IOException {
Messages.GetLiveContainersResponseProto response =
appMaster.getLiveContainers(
Messages.GetLiveContainersRequestProto.newBuilder().build());
int namesCount = response.getNamesCount();
int records = response.getContainersCount();
if (namesCount != records) {
throw new IOException("Number of names returned (" + namesCount
+ ") does not match the number of records returned: "
+ records);
}
Map<String, ContainerInformation> map = new HashMap<>(namesCount);
for (int i = 0; i < namesCount; i++) {
map.put(response.getNames(i), unmarshall(response.getContainers(i)));
}
return map;
}
public ContainerInformation getContainer(String containerId) throws
IOException {
Messages.ContainerInformationProto response =
appMaster.getLiveContainer(
Messages.GetLiveContainerRequestProto.newBuilder()
.setContainerId(containerId)
.build());
return unmarshall(response);
}
public List<ContainerInformation> getContainers() throws IOException {
Messages.GetLiveContainersResponseProto response = appMaster
.getLiveContainers(Messages.GetLiveContainersRequestProto.newBuilder()
.build());
return unmarshall(response);
}
public Map<String, ComponentInformation> enumComponents() throws IOException {
Messages.GetLiveComponentsResponseProto response =
appMaster.getLiveComponents(
Messages.GetLiveComponentsRequestProto.newBuilder().build());
int namesCount = response.getNamesCount();
int records = response.getComponentsCount();
if (namesCount != records) {
throw new IOException(
"Number of names returned (" + namesCount + ")" +
" does not match the number of records returned: " + records);
}
Map<String, ComponentInformation> map = new HashMap<>(namesCount);
for (int i = 0; i < namesCount; i++) {
map.put(response.getNames(i), unmarshall(response.getComponents(i)));
}
return map;
}
public ComponentInformation getComponent(String componentName)
throws IOException {
Messages.GetLiveComponentRequestProto.Builder builder =
Messages.GetLiveComponentRequestProto.newBuilder();
builder.setName(componentName);
Messages.ComponentInformationProto proto = appMaster.getLiveComponent(builder.build());
return unmarshall(proto);
}
public NodeInformationList getLiveNodes() throws IOException {
Messages.GetLiveNodesResponseProto response =
appMaster.getLiveNodes(Messages.GetLiveNodesRequestProto.newBuilder().build());
int records = response.getNodesCount();
NodeInformationList nil = new NodeInformationList(records);
for (int i = 0; i < records; i++) {
nil.add(unmarshall(response.getNodes(i)));
}
return nil;
}
public NodeInformation getLiveNode(String hostname) throws IOException {
Messages.GetLiveNodeRequestProto.Builder builder =
Messages.GetLiveNodeRequestProto.newBuilder();
builder.setName(hostname);
return unmarshall(appMaster.getLiveNode(builder.build()));
}
public PingInformation ping(String text) throws IOException {
return null;
}
public void stop(String text) throws IOException {
amSuicide(text, 3, 0);
}
public ApplicationLivenessInformation getApplicationLiveness() throws
IOException {
Messages.ApplicationLivenessInformationProto proto =
appMaster.getLivenessInformation(
Messages.GetApplicationLivenessRequestProto.newBuilder().build()
);
return unmarshall(proto);
}
public byte[] getClientCertificateStore(String hostname, String clientId,
String password, String type) throws IOException {
Messages.GetCertificateStoreRequestProto.Builder
builder = Messages.GetCertificateStoreRequestProto.newBuilder();
if (hostname != null) {
builder.setHostname(hostname);
}
Messages.GetCertificateStoreRequestProto requestProto =
builder.setRequesterId(clientId)
.setPassword(password)
.setType(type)
.build();
Messages.GetCertificateStoreResponseProto response =
appMaster.getClientCertificateStore(requestProto);
return unmarshall(response);
}
}

152
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java Normal file
View File

@ -0,0 +1,152 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.client.rest;
import com.google.common.base.Preconditions;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import org.apache.slider.core.exceptions.ExceptionConverter;
import org.apache.slider.core.restclient.HttpVerb;
import org.apache.slider.core.restclient.UgiJerseyBinding;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.core.MediaType;
import java.io.IOException;
import java.net.URI;
/**
* This is a base class for Jersey REST clients in Slider.
* It supports the execution of operations —with
* exceptions uprated to IOExceptions when needed.
* <p>
* Subclasses can use these operations to provide an API-like view
* of the REST model
*/
public class BaseRestClient {
private static final Logger log =
LoggerFactory.getLogger(BaseRestClient.class);
private final Client client;
public BaseRestClient(
Client client) {
Preconditions.checkNotNull(client, "null jersey client");
this.client = client;
}
/**
* Get the jersey client
* @return jersey client
*/
public Client getClient() {
return client;
}
/**
* Execute the operation. Failures are raised as IOException subclasses
* @param method method to execute
* @param resource resource to work against
* @param c class to build
* @param <T> type expected
* @return an instance of the type T
* @throws IOException on any failure
*/
public <T> T exec(HttpVerb method, WebResource resource, Class<T> c)
throws IOException {
try {
Preconditions.checkArgument(c != null);
log.debug("{}} {}", method, resource.getURI());
return resource.accept(MediaType.APPLICATION_JSON_TYPE)
.method(method.getVerb(), c);
} catch (ClientHandlerException ex) {
throw ExceptionConverter.convertJerseyException(method.getVerb(),
resource.getURI().toString(),
ex);
} catch (UniformInterfaceException ex) {
throw UgiJerseyBinding.uprateFaults(method,
resource.getURI().toString(),
ex);
}
}
/**
* Execute the operation. Failures are raised as IOException subclasses
* @param method method to execute
* @param resource resource to work against
* @param t type to work with
* @param <T> type expected
* @return an instance of the type T
* @throws IOException on any failure
*/
public <T> T exec(HttpVerb method, WebResource resource, GenericType<T> t)
throws IOException {
try {
Preconditions.checkArgument(t != null);
log.debug("{}} {}", method, resource.getURI());
resource.accept(MediaType.APPLICATION_JSON_TYPE);
return resource.method(method.getVerb(), t);
} catch (ClientHandlerException ex) {
throw ExceptionConverter.convertJerseyException(method.getVerb(),
resource.getURI().toString(),
ex);
} catch (UniformInterfaceException ex) {
throw UgiJerseyBinding.uprateFaults(method, resource.getURI().toString(),
ex);
}
}
/**
* Execute the GET operation. Failures are raised as IOException subclasses
* @param resource resource to work against
* @param c class to build
* @param <T> type expected
* @return an instance of the type T
* @throws IOException on any failure
*/
public <T> T get(WebResource resource, Class<T> c) throws IOException {
return exec(HttpVerb.GET, resource, c);
}
/**
* Create a Web resource from the client.
*
* @param u the URI of the resource.
* @return the Web resource.
*/
public WebResource resource(URI u) {
return client.resource(u);
}
/**
* Create a Web resource from the client.
*
* @param u the URI of the resource.
* @return the Web resource.
*/
public WebResource resource(String url) {
return client.resource(url);
}
}

89
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java Normal file
View File

@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.client.rest;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.WebResource;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.client.ClientRegistryBinder;
import org.apache.slider.api.SliderApplicationApi;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
import java.io.IOException;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_APPLICATION;
/**
* Factory for the Rest client; hides the lookup and instantiation.
* <p>
*
*/
public class RestClientFactory {
private final ClientRegistryBinder binder;
private final Client jerseyClient;
private final String user, serviceclass, instance;
public RestClientFactory(RegistryOperations operations,
Client jerseyClient,
String user,
String serviceclass,
String instance) {
this.jerseyClient = jerseyClient;
this.user = user;
this.serviceclass = serviceclass;
this.instance = instance;
binder = new ClientRegistryBinder(operations);
}
/**
* Locate the AM
* @return a resource to the AM
* @throws IOException any failure to resolve to the AM
*/
private WebResource locateAppmaster() throws IOException {
String restAPI = binder.lookupExternalRestAPI(user, serviceclass, instance,
CustomRegistryConstants.AM_REST_BASE);
return jerseyClient.resource(restAPI);
}
/**
* Locate the slider AM then instantiate a client instance against
* its Application API.
* @return the instance
* @throws IOException on any failure
*/
public SliderApplicationApi createSliderAppApiClient() throws IOException {
WebResource appmaster = locateAppmaster();
return createSliderAppApiClient(appmaster);
}
/**
* Create a Slider application API client instance against
* its Application API.
* @param appmaster The AM to work against.
* @return the instance
* @throws IOException on any failure
*/
public SliderApplicationApi createSliderAppApiClient(WebResource appmaster) {
WebResource appResource = appmaster.path(SLIDER_PATH_APPLICATION);
return new SliderApplicationApiRestClient(jerseyClient, appResource);
}
}

326
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java Normal file
View File

@ -0,0 +1,326 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.client.rest;
import com.google.common.base.Preconditions;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.representation.Form;
import org.apache.commons.lang.StringUtils;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.SliderApplicationApi;
import org.apache.slider.api.types.NodeInformation;
import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.ExceptionConverter;
import org.apache.slider.core.restclient.HttpVerb;
import org.apache.slider.api.types.PingInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.core.MediaType;
import java.io.IOException;
import java.util.Map;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
/**
* Implementation of the {@link SliderApplicationApi}
*/
public class SliderApplicationApiRestClient extends BaseRestClient
implements SliderApplicationApi {
private static final Logger log =
LoggerFactory.getLogger(SliderApplicationApiRestClient.class);
private WebResource appResource;
/**
* Create an instance
* @param jerseyClient jersey client for operations
* @param appResource resource of application API
*/
public SliderApplicationApiRestClient(Client jerseyClient,
WebResource appResource) {
super(jerseyClient);
this.appResource = appResource;
}
/**
* Create an instance
* @param jerseyClient jersey client for operations
* @param appmaster URL of appmaster/proxy to AM
*/
public SliderApplicationApiRestClient(Client jerseyClient, String appmaster) {
super(jerseyClient);
WebResource amResource = jerseyClient.resource(appmaster);
amResource.type(MediaType.APPLICATION_JSON);
this.appResource = amResource.path(SLIDER_PATH_APPLICATION);
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("SliderApplicationApiRestClient{");
sb.append("appResource=").append(appResource);
sb.append('}');
return sb.toString();
}
/**
* Create a resource under the application path
* @param subpath path under application
* @return a resource under the application path
*/
public WebResource applicationResource(String subpath) {
Preconditions.checkArgument(!StringUtils.isEmpty(subpath),
"empty path");
Preconditions.checkNotNull(appResource, "Null app resource");
return appResource.path(subpath);
}
/**
* Get operation against a path under the Application
* @param <T> type expected
* @param subpath path
* @param c class to instantiate
* @return instance
* @throws IOException on any problem
*/
public <T> T getApplicationResource(String subpath, Class<T> c)
throws IOException {
return appResourceOperation(HttpVerb.GET, subpath, c);
}
/**
* Get operation against a path under the Application
* @param <T> type expected
* @param subpath path
* @param t type info
* @return instance
* @throws IOException on any problem
*/
public <T> T getApplicationResource(String subpath, GenericType<T> t)
throws IOException {
return appResourceOperation(HttpVerb.GET, subpath, t);
}
/**
*
* @param method method to exec
* @param <T> type expected
* @param subpath path
* @param c class to instantiate
* @return instance
* @throws IOException on any problem
*/
public <T> T appResourceOperation(HttpVerb method, String subpath, Class<T> c)
throws IOException {
return exec(method, applicationResource(subpath), c);
}
/**
* Get operation against a path under the Application
* @param <T> type expected
* @param subpath path
* @param t type info
* @return instance
* @throws IOException on any problem
*/
public <T> T appResourceOperation(HttpVerb method, String subpath,
GenericType<T> t)
throws IOException {
return exec(method, applicationResource(subpath), t);
}
@Override
public AggregateConf getDesiredModel() throws IOException {
return getApplicationResource(MODEL_DESIRED, AggregateConf.class);
}
@Override
public ConfTreeOperations getDesiredAppconf() throws IOException {
ConfTree resource =
getApplicationResource(MODEL_DESIRED_APPCONF, ConfTree.class);
return new ConfTreeOperations(resource);
}
@Override
public ConfTreeOperations getDesiredResources() throws IOException {
ConfTree resource =
getApplicationResource(MODEL_DESIRED_RESOURCES, ConfTree.class);
return new ConfTreeOperations(resource);
}
@Override
public void putDesiredResources(ConfTree updated) throws IOException {
WebResource resource = applicationResource(MODEL_DESIRED_RESOURCES);
try {
// put operation. The result is discarded; it does help validate
// that the operation returned a JSON data structure as well as a 200
// response.
resource.accept(MediaType.APPLICATION_JSON_TYPE)
.type(MediaType.APPLICATION_JSON_TYPE)
.entity(updated)
.put(ConfTree.class);
} catch (ClientHandlerException ex) {
throw ExceptionConverter.convertJerseyException("PUT",
resource.getURI().toString(),
ex);
} catch (UniformInterfaceException ex) {
throw ExceptionConverter.convertJerseyException("PUT",
resource.getURI().toString(), ex);
}
}
@Override
public AggregateConf getResolvedModel() throws IOException {
return getApplicationResource(MODEL_RESOLVED, AggregateConf.class);
}
@Override
public ConfTreeOperations getResolvedAppconf() throws IOException {
ConfTree resource =
getApplicationResource(MODEL_RESOLVED_APPCONF, ConfTree.class);
return new ConfTreeOperations(resource);
}
@Override
public ConfTreeOperations getResolvedResources() throws IOException {
ConfTree resource =
getApplicationResource(MODEL_RESOLVED_RESOURCES, ConfTree.class);
return new ConfTreeOperations(resource);
}
@Override
public ConfTreeOperations getLiveResources() throws IOException {
ConfTree resource =
getApplicationResource(LIVE_RESOURCES, ConfTree.class);
return new ConfTreeOperations(resource);
}
@Override
public Map<String, ContainerInformation> enumContainers() throws
IOException {
return getApplicationResource(LIVE_CONTAINERS,
new GenericType<Map<String, ContainerInformation>>() {
});
}
@Override
public ContainerInformation getContainer(String containerId) throws
IOException {
return getApplicationResource(LIVE_CONTAINERS + "/" + containerId,
ContainerInformation.class);
}
@Override
public Map<String, ComponentInformation> enumComponents() throws
IOException {
return getApplicationResource(LIVE_COMPONENTS,
new GenericType<Map<String, ComponentInformation>>() { });
}
@Override
public ComponentInformation getComponent(String componentName) throws
IOException {
return getApplicationResource(LIVE_COMPONENTS + "/" + componentName,
ComponentInformation.class);
}
@Override
public NodeInformationList getLiveNodes() throws IOException {
return getApplicationResource(LIVE_NODES, NodeInformationList.class);
}
@Override
public NodeInformation getLiveNode(String hostname) throws IOException {
return getApplicationResource(LIVE_NODES + "/" + hostname,
NodeInformation.class);
}
@Override
public PingInformation ping(String text) throws IOException {
return pingPost(text);
}
/**
* Ping as a GET
* @param text text to include
* @return the response
* @throws IOException on any failure
*/
public PingInformation pingGet(String text) throws IOException {
WebResource pingResource = applicationResource(ACTION_PING);
pingResource.getUriBuilder().queryParam("body", text);
return pingResource.get(PingInformation.class);
}
/**
* Ping as a POST
* @param text text to include
* @return the response
* @throws IOException on any failure
*/
public PingInformation pingPost(String text) throws IOException {
WebResource pingResource = applicationResource(ACTION_PING);
Form f = new Form();
f.add("text", text);
return pingResource
.type(MediaType.APPLICATION_JSON_TYPE)
.post(PingInformation.class, f);
}
/**
* Ping as a POST
* @param text text to include
* @return the response
* @throws IOException on any failure
*/
public PingInformation pingPut(String text) throws IOException {
WebResource pingResource = applicationResource(ACTION_PING);
Form f = new Form();
return pingResource
.type(MediaType.TEXT_PLAIN)
.put(PingInformation.class, text);
}
@Override
public void stop(String text) throws IOException {
WebResource resource = applicationResource(ACTION_STOP);
resource.post(text);
}
@Override
public ApplicationLivenessInformation getApplicationLiveness() throws IOException {
return getApplicationResource(LIVE_LIVENESS,
ApplicationLivenessInformation.class);
}
}

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/Constants.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common;
public class Constants {
public static final int CONNECT_TIMEOUT = 10000;
public static final int RPC_TIMEOUT = 15000;
public static final String HADOOP_JAAS_DEBUG = "HADOOP_JAAS_DEBUG";
public static final String KRB5_CCNAME = "KRB5CCNAME";
public static final String JAVA_SECURITY_KRB5_CONF
= "java.security.krb5.conf";
public static final String JAVA_SECURITY_KRB5_REALM
= "java.security.krb5.realm";
public static final String SUN_SECURITY_KRB5_DEBUG
= "sun.security.krb5.debug";
public static final String SUN_SECURITY_SPNEGO_DEBUG
= "sun.security.spnego.debug";
}

88
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java Normal file
View File

@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common;
import org.apache.slider.core.main.LauncherExitCodes;
public interface SliderExitCodes extends LauncherExitCodes {
/**
* starting point for exit codes; not an exception itself
*/
int _EXIT_CODE_BASE = 64;
/**
* service entered the failed state: {@value}
*/
int EXIT_YARN_SERVICE_FAILED = 65;
/**
* service was killed: {@value}
*/
int EXIT_YARN_SERVICE_KILLED = 66;
/**
* timeout on monitoring client: {@value}
*/
int EXIT_TIMED_OUT = 67;
/**
* service finished with an error: {@value}
*/
int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 68;
/**
* the application instance is unknown: {@value}
*/
int EXIT_UNKNOWN_INSTANCE = 69;
/**
* the application instance is in the wrong state for that operation: {@value}
*/
int EXIT_BAD_STATE = 70;
/**
* A spawned master process failed
*/
int EXIT_PROCESS_FAILED = 71;
/**
* The instance failed -too many containers were
* failing or some other threshold was reached
*/
int EXIT_DEPLOYMENT_FAILED = 72;
/**
* The application is live -and the requested operation
* does not work if the cluster is running
*/
int EXIT_APPLICATION_IN_USE = 73;
/**
* There already is an application instance of that name
* when an attempt is made to create a new instance
*/
int EXIT_INSTANCE_EXISTS = 75;
/**
* Exit code when the configurations in valid/incomplete: {@value}
*/
int EXIT_BAD_CONFIGURATION = 77;
}

278
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java Normal file
View File

@ -0,0 +1,278 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common;
/**
* Keys and various constants for Slider
*/
public interface SliderKeys extends SliderXmlConfKeys {
/**
* This is the name of the slider appmaster in configurations :{@value}
*/
String COMPONENT_AM = "slider-appmaster";
/**
* Slider role is "special":{@value}
*/
int ROLE_AM_PRIORITY_INDEX = 0;
/**
* The path under which cluster and temp data are stored
* {@value}
*/
String SLIDER_BASE_DIRECTORY = ".slider";
/**
* The paths under which Slider AM dependency libraries are stored
*/
String SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK = "slider_dep";
String SLIDER_DEPENDENCY_HDP_PARENT_DIR = "/hdp";
String SLIDER_DEPENDENCY_DIR = "/apps/%s/slider";
String SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME = "slider";
String SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz";
String SLIDER_DEPENDENCY_DIR_PERMISSIONS = "755";
/**
*
*/
String HDP_VERSION_PROP_NAME = "HDP_VERSION";
/**
* name of the relative path to expaned an image into: {@value}.
* The title of this path is to help people understand it when
* they see it in their error messages
*/
String LOCAL_TARBALL_INSTALL_SUBDIR = "expandedarchive";
/**
* Application type for YARN {@value}
*/
String APP_TYPE = "org-apache-slider";
/**
* Key for application version. This must be set in app_config/global {@value}
*/
String APP_VERSION = "site.global.app_version";
String APP_VERSION_UNKNOWN = "awaiting heartbeat...";
/**
* Keys for application container specific properties, like release timeout
*/
String APP_CONTAINER_RELEASE_TIMEOUT = "site.global.app_container.release_timeout_secs";
int APP_CONTAINER_HEARTBEAT_INTERVAL_SEC = 10; // look for HEARTBEAT_IDDLE_INTERVAL_SEC
/**
* JVM arg to force IPv4 {@value}
*/
String JVM_ENABLE_ASSERTIONS = "-ea";
/**
* JVM arg enable JVM system/runtime {@value}
*/
String JVM_ENABLE_SYSTEM_ASSERTIONS = "-esa";
/**
* JVM arg to force IPv4 {@value}
*/
String JVM_FORCE_IPV4 = "-Djava.net.preferIPv4Stack=true";
/**
* JVM arg to go headless {@value}
*/
String JVM_JAVA_HEADLESS = "-Djava.awt.headless=true";
/**
* This is the name of the dir/subdir containing
* the hbase conf that is propagated via YARN
* {@value}
*/
String PROPAGATED_CONF_DIR_NAME = "propagatedconf";
String INFRA_DIR_NAME = "infra";
String GENERATED_CONF_DIR_NAME = "generated";
String SNAPSHOT_CONF_DIR_NAME = "snapshot";
String DATA_DIR_NAME = "database";
String HISTORY_DIR_NAME = "history";
String HISTORY_FILENAME_SUFFIX = "json";
String HISTORY_FILENAME_PREFIX = "rolehistory-";
String KEYTAB_DIR = "keytabs";
String RESOURCE_DIR = "resources";
/**
* Filename pattern is required to save in strict temporal order.
* Important: older files must sort less-than newer files when using
* case-sensitive name sort.
*/
String HISTORY_FILENAME_CREATION_PATTERN = HISTORY_FILENAME_PREFIX +"%016x."+
HISTORY_FILENAME_SUFFIX;
/**
* The posix regexp used to locate this
*/
String HISTORY_FILENAME_MATCH_PATTERN = HISTORY_FILENAME_PREFIX +"[0-9a-f]+\\."+
HISTORY_FILENAME_SUFFIX;
/**
* The posix regexp used to locate this
*/
String HISTORY_FILENAME_GLOB_PATTERN = HISTORY_FILENAME_PREFIX +"*."+
HISTORY_FILENAME_SUFFIX;
/**
* XML resource listing the standard Slider providers
* {@value}
*/
String SLIDER_XML = "org/apache/slider/slider.xml";
String CLUSTER_DIRECTORY = "cluster";
String PACKAGE_DIRECTORY = "package";
/**
* JVM property to define the slider configuration directory;
* this is set by the slider script: {@value}
*/
String PROPERTY_CONF_DIR = "slider.confdir";
/**
* JVM property to define the slider lib directory;
* this is set by the slider script: {@value}
*/
String PROPERTY_LIB_DIR = "slider.libdir";
/**
* name of generated dir for this conf: {@value}
*/
String SUBMITTED_CONF_DIR = "confdir";
/**
* Slider AM log4j file name : {@value}
*/
String LOG4J_SERVER_PROP_FILENAME = "log4j-server.properties";
/**
* Standard log4j file name : {@value}
*/
String LOG4J_PROP_FILENAME = "log4j.properties";
/**
* Log4j sysprop to name the resource :{@value}
*/
String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
/**
* sysprop for Slider AM log4j directory :{@value}
*/
String SYSPROP_LOG_DIR = "LOG_DIR";
/**
* name of the Slider client resource
* loaded when the service is loaded.
*/
String SLIDER_CLIENT_XML = "slider-client.xml";
/**
* The name of the resource to put on the classpath
*/
String SLIDER_SERVER_XML = "slider-server.xml";
String TMP_LOGDIR_PREFIX = "/tmp/slider-";
String TMP_DIR_PREFIX = "tmp";
String AM_DIR_PREFIX = "appmaster";
/**
* Store the default app definition, e.g. metainfo file or content of a folder
*/
String APP_DEF_DIR = "appdef";
/**
* Store additional app defs - co-processors
*/
String ADDONS_DIR = "addons";
String SLIDER_JAR = "slider.jar";
String JCOMMANDER_JAR = "jcommander.jar";
String GSON_JAR = "gson.jar";
String AGENT_TAR = "slider-agent.tar.gz";
String DEFAULT_APP_PKG = "appPkg.zip";
String DEFAULT_JVM_HEAP = "256M";
int DEFAULT_YARN_MEMORY = 256;
String STDOUT_AM = "slider-out.txt";
String STDERR_AM = "slider-err.txt";
String DEFAULT_GC_OPTS = "";
String HADOOP_USER_NAME = "HADOOP_USER_NAME";
String HADOOP_PROXY_USER = "HADOOP_PROXY_USER";
String SLIDER_PASSPHRASE = "SLIDER_PASSPHRASE";
boolean PROPAGATE_RESOURCE_OPTION = true;
/**
* Security associated keys.
*/
String SECURITY_DIR = "security";
String CRT_FILE_NAME = "ca.crt";
String CSR_FILE_NAME = "ca.csr";
String KEY_FILE_NAME = "ca.key";
String KEYSTORE_FILE_NAME = "keystore.p12";
String CRT_PASS_FILE_NAME = "pass.txt";
String PASS_LEN = "50";
String COMP_STORES_REQUIRED_KEY =
"slider.component.security.stores.required";
String COMP_KEYSTORE_PASSWORD_PROPERTY_KEY =
"slider.component.keystore.password.property";
String COMP_KEYSTORE_PASSWORD_ALIAS_KEY =
"slider.component.keystore.credential.alias.property";
String COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT =
"component.keystore.credential.alias";
String COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY =
"slider.component.truststore.password.property";
String COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY =
"slider.component.truststore.credential.alias.property";
String COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT =
"component.truststore.credential.alias";
/**
* Python specific
*/
String PYTHONPATH = "PYTHONPATH";
/**
* Name of the AM filter to use: {@value}
*/
String AM_FILTER_NAME =
"org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer";
/**
* Allowed port range. This MUST be set in app_conf/global.
* {@value}
*/
String KEY_ALLOWED_PORT_RANGE = "site.global.slider.allowed.ports";
/**
* env var for custom JVM options.
*/
String SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS";
String SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA";
String YARN_CONTAINER_PATH = "/node/container/";
}

83
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java Normal file
View File

@ -0,0 +1,83 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common;
/**
* Keys shared across tests
*/
public interface SliderXMLConfKeysForTesting {
String KEY_TEST_THAW_WAIT_TIME = "slider.test.thaw.wait.seconds";
int DEFAULT_THAW_WAIT_TIME_SECONDS = 60;
String KEY_TEST_FREEZE_WAIT_TIME = "slider.test.freeze.wait.seconds";
int DEFAULT_TEST_FREEZE_WAIT_TIME_SECONDS = 60;
String KEY_TEST_TIMEOUT = "slider.test.timeout.seconds";
int DEFAULT_TEST_TIMEOUT_SECONDS = 30 * 60;
String KEY_ACCUMULO_LAUNCH_TIME =
"slider.test.accumulo.launch.wait.seconds";
int DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS = 60 * 3;
String KEY_ACCUMULO_GO_LIVE_TIME =
"slider.test.accumulo.live.wait.seconds";
int DEFAULT_ACCUMULO_LIVE_TIME_SECONDS = 90;
String KEY_TEST_AGENT_ENABLED = "slider.test.agent.enabled";
String KEY_AGENTTESTS_QUEUE_LABELED_DEFINED = "slider.test.agent.labeled.queue.enabled";
String KEY_AGENTTESTS_LABELS_RED_BLUE_DEFINED = "slider.test.agent.labels.defined";
String KEY_AGENTTESTS_AM_FAILURES_ENABLED = "slider.test.agent.am.failures.enabled";
int DEFAULT_AGENT_LAUNCH_TIME_SECONDS = 60 * 3;
String KEY_TEST_AGENT_HOME = "slider.test.agent.home";
String KEY_TEST_AGENT_TAR = "slider.test.agent.tar";
String KEY_TEST_TEARDOWN_KILLALL = "slider.test.teardown.killall";
boolean DEFAULT_TEARDOWN_KILLALL = true;
/**
* Key for amount of RAM to request
*/
String KEY_TEST_YARN_RAM_REQUEST = "slider.test.yarn.ram";
String DEFAULT_YARN_RAM_REQUEST = "192";
/**
* security related keys
*/
String TEST_SECURITY_DIR = "/tmp/work/security";
/**
* Local path to AM keytab: {@value}
*/
String KEY_TEST_AM_KEYTAB = "slider.test.am.keytab.local";
/**
* Is the test cluster windows? Default is: same as the local system.
* {@value}
*/
String KEY_TEST_WINDOWS_CLUSTER = "slider.test.windows.cluster";
}

206
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java Normal file
View File

@ -0,0 +1,206 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common;
import org.apache.hadoop.registry.client.api.RegistryConstants;
/**
* These are the keys that can be added to <code>conf/slider-client.xml</code>.
*/
public interface SliderXmlConfKeys {
String PREFIX_PROVIDER = "slider.provider";
/**
* pattern to identify a provider
* {@value}
*/
String KEY_PROVIDER = PREFIX_PROVIDER + ".%s";
/**
* conf option set to point to where the config came from
* {@value}
*/
String KEY_TEMPLATE_ORIGIN = "slider.template.origin";
/**
* Original name for the default FS. This is still
* expected by applications deployed
*/
String FS_DEFAULT_NAME_CLASSIC = "fs.default.name";
/**
* Slider principal
*/
String KEY_KERBEROS_PRINCIPAL = "slider.kerberos.principal";
/**
* Name of the property for ACLs for Slider AM.
* {@value}
*/
String KEY_PROTOCOL_ACL = "slider.security.protocol.acl";
/**
* Limit on restarts for the AM
* {@value}
*/
String KEY_AM_RESTART_LIMIT = "slider.yarn.restart.limit";
/**
* queue name, by default let YARN pick the queue
*/
String KEY_YARN_QUEUE = "slider.yarn.queue";
String DEFAULT_YARN_QUEUE = null;
/**
* default priority
*/
String KEY_YARN_QUEUE_PRIORITY = "slider.yarn.queue.priority";
int DEFAULT_YARN_QUEUE_PRIORITY = 1;
/**
* The slider base path: {@value}
* Defaults to HomeDir/.slider
*/
String KEY_SLIDER_BASE_PATH = "slider.base.path";
/**
* Option for the permissions for the cluster directory itself: {@value}
*/
String CLUSTER_DIRECTORY_PERMISSIONS =
"slider.cluster.directory.permissions";
/**
* Default value for the permissions :{@value}
*/
String DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS = "750";
/**
*
* Option for the permissions for the data directory itself: {@value}
*/
String DATA_DIRECTORY_PERMISSIONS = "slider.data.directory.permissions";
/**
* Default value for the data directory permissions: {@value}
*/
String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750";
/**
*
* Use {@link RegistryConstants#KEY_REGISTRY_ZK_ROOT}
*
*/
@Deprecated
String REGISTRY_PATH = "slider.registry.path";
/**
*
* @Deprecated use {@link RegistryConstants#KEY_REGISTRY_ZK_QUORUM}
*
*/
@Deprecated
String REGISTRY_ZK_QUORUM = "slider.zookeeper.quorum";
String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH =
"ipc.client.fallback-to-simple-auth-allowed";
String HADOOP_HTTP_FILTER_INITIALIZERS =
"hadoop.http.filter.initializers";
String KEY_KEYSTORE_LOCATION = "ssl.server.keystore.location";
String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
/** Declare that a keytab must be provided */
String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
String KEY_SECURITY_ENABLED = "site.global.security_enabled";
/**
* Set to disable server-side checks for python, openssl &c.
* This should only be set for testing
*/
String KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED =
"slider.am.dependency.checks.disabled";
/**
* The path to the python executable utilized to launch the agent.
*/
String PYTHON_EXECUTABLE_PATH = "agent.python.exec.path";
/**
* Flag to enable the insecure AM filter: {@value}
*/
String X_DEV_INSECURE_WS = "slider.feature.ws.insecure";
/**
* Flag to indicate the insecure AM filter is enabled by default: {@value}.
*/
boolean X_DEV_INSECURE_DEFAULT = false;
/**
* Flag to indicate the insecure AM filter is required for
* complex REST Verbs: {@value}.
* When Slider switches to being Hadoop 2.7+ only, this flag
* can be set to false
*/
boolean X_DEV_INSECURE_REQUIRED = true;
/**
*
*/
String KEY_IPC_CLIENT_RETRY_POLICY_ENABLED =
"slider.ipc.client.retry.enabled";
boolean IPC_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = true;
String KEY_IPC_CLIENT_RETRY_POLICY_SPEC =
"slider.ipc.client.retry.policy.spec";
String IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
"10000,6,60000,10"; //t1,n1,t2,n2,...
String KEY_AM_LAUNCH_ENV = "slider.am.launch.env";
/**
* From {@code DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY}
*/
String DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY = "dfs.namenode.kerberos.principal";
String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = "dfs.datanode.kerberos.principal";
//Delegation token related keys
String DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY
= "dfs.namenode.delegation.key.update-interval";
long DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60 *
1000; // 1 day
String DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY
= "dfs.namenode.delegation.token.renew-interval";
long DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 *
1000; // 1 day
String DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY
= "dfs.namenode.delegation.token.max-lifetime";
long DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 *
1000; // 7 days
String DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY
= "dfs.namenode.delegation.token.always-use"; // for tests
boolean DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT = false;
String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file";
String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved";
String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary";
}

178
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java Normal file
View File

@ -0,0 +1,178 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import org.apache.hadoop.fs.Path;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.UsageException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
/**
* Base args for all actions
*/
public abstract class AbstractActionArgs extends ArgOps implements Arguments {
protected static final Logger log =
LoggerFactory.getLogger(AbstractActionArgs.class);
protected AbstractActionArgs() {
}
/**
* URI/binding to the filesystem
*/
@Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
description = "Filesystem Binding")
public String filesystemBinding;
@Parameter(names = {ARG_BASE_PATH},
description = "Slider base path on the filesystem",
converter = PathArgumentConverter.class)
public Path basePath;
/**
* This is the default parameter
*/
@Parameter
public final List<String> parameters = new ArrayList<>();
/**
* get the name: relies on arg 1 being the cluster name in all operations
* @return the name argument, null if there is none
*/
public String getClusterName() {
return (parameters.isEmpty()) ? null : parameters.get(0);
}
/**
-D name=value
Define an HBase configuration option which overrides any options in
the configuration XML files of the image or in the image configuration
directory. The values will be persisted.
Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
*/
@Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions")
public final List<String> definitions = new ArrayList<>();
/**
* System properties
*/
@Parameter(names = {ARG_SYSPROP}, arity = 1,
description = "system properties in the form name value" +
" These are set after the JVM is started.")
public final List<String> sysprops = new ArrayList<>(0);
@Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
description = "Binding (usually hostname:port) of the YARN resource manager")
public String manager;
@Parameter(names = ARG_DEBUG, description = "Debug mode")
public boolean debug = false;
@Parameter(names = {ARG_HELP}, description = "Help", help = true)
public boolean help = false;
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 1;
}
/**
* Get the name of the action
* @return the action name
*/
public abstract String getActionName() ;
/**
* Get the max #of params expected
* @return the number of params in the {@link #parameters} field;
*/
public int getMaxParams() {
return getMinParams();
}
public void validate() throws BadCommandArgumentsException, UsageException {
int minArgs = getMinParams();
int actionArgSize = parameters.size();
if (minArgs > actionArgSize) {
throw new BadCommandArgumentsException(
ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() +
" Expected minimum " + minArgs + " but got " + actionArgSize);
}
int maxArgs = getMaxParams();
if (maxArgs == -1) {
maxArgs = minArgs;
}
if (actionArgSize > maxArgs) {
String message = String.format("%s for action %s: limit is %d but saw %d: ",
ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
getActionName(), maxArgs,
actionArgSize);
log.error(message);
int index = 1;
for (String actionArg : parameters) {
log.error("[{}] \"{}\"", index++, actionArg);
message += " \"" + actionArg + "\" ";
}
throw new BadCommandArgumentsException(message);
}
}
@Override
public String toString() {
return super.toString() + ": " + getActionName();
}
/**
* Override point:
* Flag to indicate that core hadoop API services are needed (HDFS, YARN, etc)
* —and that validation of the client state should take place.
*
* @return a flag to indicate that the core hadoop services will be needed.
*/
public boolean getHadoopServicesRequired() {
return true;
}
/**
* Flag to disable secure login.
* This MUST only be set if the action is bypassing security or setting
* it itself
* @return true if login at slider client init time is to be skipped
*/
public boolean disableSecureLogin() {
return false;
}
}

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java Normal file
View File

@ -0,0 +1,26 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
/**
* Base class for all the delegates
*/
public class AbstractArgsDelegate extends ArgOps implements Arguments {
}

217
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java Normal file
View File

@ -0,0 +1,217 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParametersDelegate;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.Path;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.providers.SliderProviderFactory;
import java.io.File;
import java.util.List;
import java.util.Map;
/**
* Abstract Action to build things; shares args across build and
* list
*/
public abstract class AbstractClusterBuildingActionArgs extends
AbstractActionArgs {
/**
* Declare the image configuration directory to use when creating or
* reconfiguring a slider cluster. The path must be on a filesystem visible
* to all nodes in the YARN cluster. Only one configuration directory can
* be specified.
*/
@Parameter(names = ARG_CONFDIR,
description = "Path to cluster configuration directory in HDFS",
converter = PathArgumentConverter.class)
public Path confdir;
@Parameter(names = ARG_ZKPATH,
description = "Zookeeper path for the application")
public String appZKPath;
@Parameter(names = ARG_ZKHOSTS,
description = "comma separated list of the Zookeeper hosts")
public String zkhosts;
/**
* --image path
* the full path to a .tar or .tar.gz path containing an HBase image.
*/
@Parameter(names = ARG_IMAGE,
description = "The full path to a .tar or .tar.gz path containing the application",
converter = PathArgumentConverter.class)
public Path image;
@Parameter(names = ARG_APP_HOME,
description = "Home directory of a pre-installed application")
public String appHomeDir;
@Parameter(names = ARG_PROVIDER,
description = "Provider of the specific cluster application")
public String provider = SliderProviderFactory.DEFAULT_CLUSTER_TYPE;
@Parameter(names = {ARG_PACKAGE},
description = "URI to a slider package")
public String packageURI;
@Parameter(names = {ARG_RESOURCES},
description = "File defining the resources of this instance")
public File resources;
@Parameter(names = {ARG_TEMPLATE},
description = "Template application configuration")
public File template;
@Parameter(names = {ARG_METAINFO},
description = "Application meta info file")
public File appMetaInfo;
@Parameter(names = {ARG_METAINFO_JSON},
description = "Application meta info JSON blob")
public String appMetaInfoJson;
@Parameter(names = {ARG_APPDEF},
description = "Application def (folder or a zip package)")
public File appDef;
@Parameter(names = {ARG_QUEUE},
description = "Queue to submit the application")
public String queue;
@ParametersDelegate
public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
@ParametersDelegate
public AddonArgsDelegate addonDelegate = new AddonArgsDelegate();
@ParametersDelegate
public AppAndResouceOptionArgsDelegate optionsDelegate =
new AppAndResouceOptionArgsDelegate();
public Map<String, String> getOptionsMap() throws
BadCommandArgumentsException {
return optionsDelegate.getOptionsMap();
}
/**
* Get the role heap mapping (may be empty, but never null)
* @return role heap mapping
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, Map<String, String>> getCompOptionMap() throws
BadCommandArgumentsException {
return optionsDelegate.getCompOptionMap();
}
public Map<String, String> getResourceOptionsMap() throws
BadCommandArgumentsException {
return optionsDelegate.getResourceOptionsMap();
}
/**
* Get the role heap mapping (may be empty, but never null)
* @return role heap mapping
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, Map<String, String>> getResourceCompOptionMap() throws
BadCommandArgumentsException {
return optionsDelegate.getResourceCompOptionMap();
}
@VisibleForTesting
public List<String> getComponentTuples() {
return componentDelegate.getComponentTuples();
}
/**
* Get the role mapping (may be empty, but never null)
* @return role mapping
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, String> getComponentMap() throws
BadCommandArgumentsException {
return componentDelegate.getComponentMap();
}
@VisibleForTesting
public List<String> getAddonTuples() {
return addonDelegate.getAddonTuples();
}
/**
* Get the list of addons (may be empty, but never null)
*/
public Map<String, String> getAddonMap() throws
BadCommandArgumentsException {
return addonDelegate.getAddonMap();
}
public Path getConfdir() {
return confdir;
}
public String getAppZKPath() {
return appZKPath;
}
public String getZKhosts() {
return zkhosts;
}
public Path getImage() {
return image;
}
public String getAppHomeDir() {
return appHomeDir;
}
public String getProvider() {
return provider;
}
public ConfTree buildAppOptionsConfTree() throws
BadCommandArgumentsException {
return buildConfTree(getOptionsMap());
}
public ConfTree buildResourceOptionsConfTree() throws
BadCommandArgumentsException {
return buildConfTree(getResourceOptionsMap());
}
protected ConfTree buildConfTree(Map<String, String> optionsMap) throws
BadCommandArgumentsException {
ConfTree confTree = new ConfTree();
ConfTreeOperations ops = new ConfTreeOperations(confTree);
confTree.global.putAll(optionsMap);
return confTree;
}
}

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_AM_SUICIDE},
commandDescription = SliderActions.DESCRIBE_ACTION_AM_SUICIDE)
public class ActionAMSuicideArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_AM_SUICIDE;
}
@Parameter(names = {ARG_MESSAGE},
description = "reason for the action")
public String message = "";
@Parameter(names = {ARG_EXITCODE},
description = "exit code")
public int exitcode = 1;
@Parameter(names = {ARG_WAIT},
description = "time for AM to wait before exiting")
public int waittime = 1000;
}

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_BUILD},
commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_BUILD;
}
}

98
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java Normal file
View File

@ -0,0 +1,98 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import java.io.File;
@Parameters(commandNames = {SliderActions.ACTION_CLIENT},
commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT)
public class ActionClientArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_CLIENT;
}
@Parameter(names = {ARG_INSTALL},
description = "Install client")
public boolean install;
@Parameter(names = {ARG_GETCERTSTORE},
description = "Get a certificate store")
public boolean getCertStore;
@Parameter(names = {ARG_KEYSTORE},
description = "Retrieve keystore to specified location")
public File keystore;
@Parameter(names = {ARG_TRUSTSTORE},
description = "Retrieve truststore to specified location")
public File truststore;
@Parameter(names = {ARG_HOSTNAME},
description = "(Optional) Specify the hostname to use for generation of keystore certificate")
public String hostname;
@Parameter(names = {ARG_NAME},
description = "The name of the application")
public String name;
@Parameter(names = {ARG_PROVIDER},
description = "The credential provider in which the password is stored")
public String provider;
@Parameter(names = {ARG_ALIAS},
description = "The credential provider alias associated with the password")
public String alias;
@Parameter(names = {ARG_PASSWORD},
description = "The certificate store password (alternative to " +
"provider/alias; if password is specified, those will be ignored)")
public String password;
@Parameter(names = {ARG_PACKAGE},
description = "Path to app package")
public String packageURI;
@Parameter(names = {ARG_DEST},
description = "The location where to install the client")
public File installLocation;
@Parameter(names = {ARG_CONFIG},
description = "Client configuration")
public File clientConfig;
/**
* Get the min #of params expected
*
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 1;
}
}

59
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import java.io.File;
@Parameters(commandNames = {SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
public class ActionCreateArgs extends AbstractClusterBuildingActionArgs
implements WaitTimeAccessor, LaunchArgsAccessor {
@Override
public String getActionName() {
return SliderActions.ACTION_CREATE;
}
@ParametersDelegate
LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
@Override
public File getOutputFile() {
return launchArgs.getOutputFile();
}
@Override
public String getRmAddress() {
return launchArgs.getRmAddress();
}
@Override
public int getWaittime() {
return launchArgs.getWaittime();
}
@Override
public void setWaittime(int waittime) {
launchArgs.setWaittime(waittime);
}
}

65
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionDependencyArgs.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.UsageException;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = { SliderActions.ACTION_DEPENDENCY },
commandDescription = SliderActions.DESCRIBE_ACTION_DEPENDENCY)
public class ActionDependencyArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_DEPENDENCY;
}
@Parameter(names = { ARG_UPLOAD },
description = "Upload AM and agent libraries to HDFS for this client")
public boolean upload;
@Parameter(names = { ARG_OVERWRITE },
description = "Overwrite current uploaded dependency libs")
public boolean overwrite = false;
/**
* Get the min #of params expected
*
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 1;
}
@Override
public void validate() throws BadCommandArgumentsException, UsageException {
super.validate();
if (!upload) {
throw new UsageException("Option " + ARG_UPLOAD + " is mandatory");
}
}
}

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_DESTROY},
commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY)
public class ActionDestroyArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_DESTROY;
}
@Parameter(names = {ARG_FORCE},
description = "force the operation")
public boolean force;
}

73
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(
commandNames = {SliderActions.ACTION_DIAGNOSTICS},
commandDescription = SliderActions.DESCRIBE_ACTION_DIAGNOSTIC)
public class ActionDiagnosticArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_DIAGNOSTICS;
}
@Parameter(names = {ARG_NAME},
description = "the name of the running application")
public String name;
@Parameter(names = {ARG_CLIENT},
description = "print configuration of the slider client")
public boolean client = false;
@Parameter(names = {ARG_APPLICATION},
description = "print configuration of the running application")
public boolean application;
@Parameter(names = {ARG_VERBOSE},
description = "print out information in details")
public boolean verbose = false;
@Parameter(names = {ARG_YARN},
description = "print configuration of the YARN cluster")
public boolean yarn = false;
@Parameter(names = {ARG_CREDENTIALS},
description = "print credentials of the current user")
public boolean credentials = false;
@Parameter(names = {ARG_ALL},
description = "print all of the information above")
public boolean all;
@Parameter(names = {ARG_LEVEL},
description = "diagnose each slider configuration one by one")
public boolean level;
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
@Override
public int getMinParams() {
return 0;
}
}

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
public class ActionEchoArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_ECHO;
}
@Parameter(names = {ARG_MESSAGE},
description = "message to echo")
public String message;
}

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import java.io.File;
@Parameters(commandNames = {SliderActions.ACTION_EXISTS},
commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
public class ActionExistsArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_EXISTS;
}
@Parameter(names = {ARG_LIVE},
description = "verify that the application is running")
public boolean live;
@Parameter(names = {ARG_STATE},
description = "verify that the application is in the specific YARN state")
public String state = "";
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
description = "output file for any application report")
public File out;
}

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import java.util.List;
import java.util.Map;
@Parameters(commandNames = {SliderActions.ACTION_FLEX},
commandDescription = SliderActions.DESCRIBE_ACTION_FLEX)
public class ActionFlexArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_FLEX;
}
@ParametersDelegate
public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
/**
* Get the component mapping (may be empty, but never null)
* @return mapping
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
return componentDelegate.getComponentMap();
}
public List<String> getComponentTuples() {
return componentDelegate.getComponentTuples();
}
}

56
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
@Parameters(commandNames = {SliderActions.ACTION_FREEZE},
commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE)
public class ActionFreezeArgs extends AbstractActionArgs implements
WaitTimeAccessor {
@Override
public String getActionName() {
return SliderActions.ACTION_FREEZE;
}
public static final String FREEZE_COMMAND_ISSUED = "stop command issued";
@ParametersDelegate
public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
@Override
public int getWaittime() {
return waitDelegate.getWaittime();
}
@Override
public void setWaittime(int waittime) {
waitDelegate.setWaittime(waittime);
}
@Parameter(names={ARG_MESSAGE},
description = "reason for the operation")
public String message = FREEZE_COMMAND_ISSUED;
@Parameter(names = {ARG_FORCE},
description = "force the operation")
public boolean force;
}

51
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameters;
/**
* The Help command
*/
@Parameters(commandNames = {SliderActions.ACTION_HELP},
commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
public class ActionHelpArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_HELP;
}
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
@Override
public int getMinParams() {
return 0;
}
/**
* This action does not need hadoop services
* @return false
*/
@Override
public boolean getHadoopServicesRequired() {
return false;
}
}

57
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallKeytabArgs.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_INSTALL_KEYTAB},
commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_KEYTAB)
public class ActionInstallKeytabArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_INSTALL_KEYTAB;
}
@Parameter(names = {ARG_KEYTAB},
description = "Path to keytab on local disk")
public String keytabUri;
@Parameter(names = {ARG_FOLDER},
description = "The name of the folder in which to store the keytab")
public String folder;
@Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
public boolean overwrite = false;
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 3;
}
}

58
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
@Parameters(commandNames = {SliderActions.ACTION_INSTALL_PACKAGE},
commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_PACKAGE)
public class ActionInstallPackageArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_INSTALL_PACKAGE;
}
@Parameter(names = {ARG_PACKAGE},
description = "Path to app package on local disk")
public String packageURI;
@Parameter(names = {ARG_NAME},
description = "The type of the package")
public String name;
@Parameter(names = {ARG_REPLACE_PKG}, description = "Overwrite existing package")
public boolean replacePkg = false;
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 1;
}
}

86
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKDiagArgs.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.UsageException;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
@Parameters(commandNames = {SliderActions.ACTION_KDIAG},
commandDescription = SliderActions.DESCRIBE_ACTION_KDIAG)
public class ActionKDiagArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_KDIAG;
}
@Parameter(names = {ARG_SERVICES}, variableArity = true,
description =" list of services to check")
public List<String> services = new ArrayList<>();
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
description = "output file for report")
public File out;
@Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
public File keytab;
@Parameter(names = {ARG_KEYLEN}, description = "minimum key length")
public int keylen = 256;
@Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
public String principal;
@Parameter(names = {ARG_SECURE}, description = "Is security required")
public boolean secure = false;
@Override
public int getMinParams() {
return 0;
}
@Override
public boolean getHadoopServicesRequired() {
return false;
}
@Override
public boolean disableSecureLogin() {
return true;
}
@Override
public void validate() throws BadCommandArgumentsException, UsageException {
super.validate();
if (keytab != null && SliderUtils.isUnset(principal)) {
throw new UsageException("Missing argument " + ARG_PRINCIPAL);
}
if (keytab == null && SliderUtils.isSet(principal)) {
throw new UsageException("Missing argument " + ARG_KEYTAB);
}
}
}

81
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_KEYTAB},
commandDescription = SliderActions.DESCRIBE_ACTION_KEYTAB)
public class ActionKeytabArgs extends AbstractActionArgs {
public ActionKeytabArgs(ActionInstallKeytabArgs installKeytabInfo) {
this.install = true;
this.overwrite = installKeytabInfo.overwrite;
this.keytab = installKeytabInfo.keytabUri;
this.folder = installKeytabInfo.folder;
}
public ActionKeytabArgs() {
super();
}
@Override
public String getActionName() {
return SliderActions.ACTION_INSTALL_KEYTAB;
}
@Parameter(names = {ARG_KEYTABINSTALL},
description = "Install the keytab")
public boolean install;
@Parameter(names = {ARG_KEYTABDELETE},
description = "Delete the keytab")
public boolean delete;
@Parameter(names = {ARG_KEYTABLIST},
description = "List of installed keytabs")
public boolean list;
@Parameter(names = {ARG_KEYTAB},
description = "Path or name of the keytab")
public String keytab;
@Parameter(names = {ARG_FOLDER},
description = "The name of the folder in which to store the keytab")
public String folder;
@Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
public boolean overwrite = false;
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 3;
}
}

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKillContainerArgs.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_KILL_CONTAINER},
commandDescription = SliderActions.DESCRIBE_ACTION_KILL_CONTAINER)
public class ActionKillContainerArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_KILL_CONTAINER;
}
@Parameter(names = {ARG_ID},
description = "ID of the container")
public String id;
}

74
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java Normal file
View File

@ -0,0 +1,74 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import java.util.HashSet;
import java.util.Set;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_LIST},
commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
public class ActionListArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_LIST;
}
@Parameter(names = {ARG_LIVE},
description = "List only live application instances")
public boolean live;
@Parameter(names = {ARG_STATE},
description = "list only applications in the specific YARN state")
public String state = "";
@Parameter(names = {ARG_VERBOSE},
description = "print out information in details")
public boolean verbose = false;
@Parameter(names = {ARG_CONTAINERS},
description = "List containers of an application instance")
public boolean containers;
@Parameter(names = {ARG_VERSION},
description = "Filter containers by app version (used with " +
ARG_CONTAINERS + ")")
public String version;
@Parameter(names = {ARG_COMPONENTS}, variableArity = true,
description = "Filter containers by component names (used with " +
ARG_CONTAINERS + ")")
public Set<String> components = new HashSet<>(0);
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 1;
}
}

76
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java Normal file
View File

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.commons.lang.StringUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.UsageException;
import java.io.File;
@Parameters(commandNames = {SliderActions.ACTION_LOOKUP},
commandDescription = SliderActions.DESCRIBE_ACTION_LOOKUP)
public class ActionLookupArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_LOOKUP;
}
public int getMinParams() {
return 0;
}
public int getMaxParams() {
return 0;
}
@Parameter(names = {ARG_ID},
description = "ID of the application")
public String id;
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
description = "output file for any application report")
public File outputFile;
@Override
public void validate() throws BadCommandArgumentsException, UsageException {
super.validate();
if (StringUtils.isEmpty(id)) {
throw new BadCommandArgumentsException("Missing mandatory argument "
+ ARG_ID);
}
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder(SliderActions.ACTION_LOOKUP);
if (id!=null) {
sb.append(" ");
sb.append(ARG_ID).append(" ").append(id);
}
if (outputFile != null) {
sb.append(" ");
sb.append(ARG_OUTPUT).append(" ").append(outputFile.getAbsolutePath());
}
return sb.toString();
}
}

71
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import java.io.File;
@Parameters(commandNames = {SliderActions.ACTION_NODES},
commandDescription = SliderActions.DESCRIBE_ACTION_NODES)
public class ActionNodesArgs extends AbstractActionArgs {
/**
* Instance for API use; on CLI the name is derived from {@link #getClusterName()}.
*/
public String instance;
@Override
public String getActionName() {
return SliderActions.ACTION_NODES;
}
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
description = "Output file for the information")
public File outputFile;
@Parameter(names = {ARG_LABEL})
public String label = "";
@Parameter(names = {ARG_HEALTHY} )
public boolean healthy;
@Override
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 1;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder(
"ActionNodesArgs{");
sb.append("instance='").append(instance).append('\'');
sb.append(", outputFile=").append(outputFile);
sb.append(", label='").append(label).append('\'');
sb.append(", healthy=").append(healthy);
sb.append('}');
return sb.toString();
}
}

81
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_PACKAGE},
commandDescription = SliderActions.DESCRIBE_ACTION_PACKAGE)
public class ActionPackageArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_PACKAGE;
}
@Parameter(names = {ARG_INSTALL},
description = "Install package in the sub-folder 'package' of the user's Slider base directory")
public boolean install;
@Parameter(names = {ARG_PKGDELETE},
description = "Delete package operation")
public boolean delete;
@Parameter(names = {ARG_PKGLIST},
description = "List of package(s) installed")
public boolean list;
@Parameter(names = {ARG_PKGINSTANCES},
description = "Lists all application instances referring to package")
public boolean instances;
@Parameter(names = {ARG_PACKAGE},
description = "Path to app package on local disk")
public String packageURI;
@Parameter(names = {ARG_NAME},
description = "Package name")
public String name;
@Parameter(names = {ARG_VERSION}, description = "Package version")
public String version;
@Parameter(names = {ARG_REPLACE_PKG},
description = "Overwrite existing package")
public boolean replacePkg = false;
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
description = "Output file for package data")
public String out;
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 1;
}
}

218
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java Normal file
View File

@ -0,0 +1,218 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.UsageException;
import org.apache.slider.core.registry.docstore.ConfigFormat;
import static org.apache.slider.common.params.SliderActions.ACTION_REGISTRY;
import static org.apache.slider.common.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
import java.io.File;
/**
* Registry actions
*
* --instance {app name}, if a / is in it, refers underneath?
* --dest {destfile}
* --list : list instances of slider service
* --listfiles
*/
@Parameters(commandNames = {ACTION_REGISTRY},
commandDescription = DESCRIBE_ACTION_REGISTRY)
public class ActionRegistryArgs extends AbstractActionArgs {
public static final String USAGE =
"Usage: " + SliderActions.ACTION_REGISTRY
+ " ("
+ Arguments.ARG_LIST + "|"
+ Arguments.ARG_LISTCONF + "|"
+ Arguments.ARG_LISTEXP + "|"
+ Arguments.ARG_LISTFILES + "|"
+ Arguments.ARG_GETCONF + "|"
+ Arguments.ARG_GETEXP + "> "
+ Arguments.ARG_NAME + " <name> "
+ " )"
+ "[" + Arguments.ARG_VERBOSE + "] "
+ "[" + Arguments.ARG_USER + "] "
+ "[" + Arguments.ARG_OUTPUT + " <filename> ] "
+ "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
+ "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
+ System.getProperty("line.separator")
+ "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
;
public ActionRegistryArgs() {
}
public ActionRegistryArgs(String name) {
this.name = name;
}
@Override
public String getActionName() {
return ACTION_REGISTRY;
}
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
@Override
public int getMinParams() {
return 0;
}
@Parameter(names = {ARG_LIST},
description = "list services")
public boolean list;
@Parameter(names = {ARG_LISTCONF},
description = "list configurations")
public boolean listConf;
@Parameter(names = {ARG_GETCONF},
description = "get configuration")
public String getConf;
@Parameter(names = {ARG_LISTEXP},
description = "list exports")
public boolean listExports;
@Parameter(names = {ARG_GETEXP},
description = "get export")
public String getExport;
@Parameter(names = {ARG_LISTFILES},
description = "list files")
public String listFiles;
@Parameter(names = {ARG_GETFILES},
description = "get files")
public String getFiles;
//--format
@Parameter(names = ARG_FORMAT,
description = "Format for a response: <xml|json|properties>")
public String format = ConfigFormat.XML.toString() ;
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
description = "Output destination")
public File out;
@Parameter(names = {ARG_NAME},
description = "name of an instance")
public String name;
@Parameter(names = {ARG_SERVICETYPE},
description = "optional service type")
public String serviceType = SliderKeys.APP_TYPE;
@Parameter(names = {ARG_VERBOSE},
description = "verbose output")
public boolean verbose;
@Parameter(names = {ARG_INTERNAL},
description = "fetch internal registry entries")
public boolean internal;
@Parameter(names = {ARG_USER},
description = "the name of the user whose application is being resolved")
public String user;
/**
* validate health of all the different operations
* @throws BadCommandArgumentsException
*/
@Override
public void validate() throws BadCommandArgumentsException, UsageException {
super.validate();
//verify that at most one of the operations is set
int gets = s(getConf) + s(getFiles) + s(getExport);
int lists = s(list) + s(listConf) + s(listFiles) + s(listExports);
int set = lists + gets;
if (set > 1) {
throw new UsageException(USAGE);
}
if (out != null && ( set == 0)) {
throw new UsageException("output path"
+ " is only supported on 'get' operations: ");
}
if (!list && !is(name)) {
throw new UsageException("Argument " + ARG_NAME
+" missing: ");
}
}
private int s(String arg) {
return is(arg) ? 1 : 0;
}
private boolean is(String arg) {
return arg != null;
}
private int s(boolean arg) {
return arg ? 1 : 0;
}
private String ifdef(String arg, boolean val) {
return val ? (arg + " "): "";
}
private String ifdef(String arg, String val) {
if (is(val)) {
return arg + " " + val + " ";
} else {
return "";
}
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder(ACTION_REGISTRY);
sb.append(' ');
sb.append(ifdef(ARG_LIST, list));
sb.append(ifdef(ARG_LISTCONF, listConf));
sb.append(ifdef(ARG_LISTFILES, listFiles));
sb.append(ifdef(ARG_GETCONF, getConf));
sb.append(ifdef(ARG_GETFILES, getFiles));
sb.append(ifdef(ARG_NAME, name));
sb.append(ifdef(ARG_SERVICETYPE, serviceType));
sb.append(ifdef(ARG_VERBOSE, verbose));
sb.append(ifdef(ARG_INTERNAL, internal));
if (out != null) {
sb.append(ifdef(ARG_OUTPUT, out.toString()));
}
sb.append(ifdef(ARG_FORMAT, format));
return sb.toString();
}
}

153
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java Normal file
View File

@ -0,0 +1,153 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.commons.lang.StringUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.UsageException;
import java.io.File;
import static org.apache.slider.common.params.SliderActions.ACTION_RESOLVE;
import static org.apache.slider.common.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
/**
* Resolve registry entries
*
* --path {path}
* --out {destfile}
* --verbose
* --list
*/
@Parameters(commandNames = {ACTION_RESOLVE},
commandDescription = DESCRIBE_ACTION_REGISTRY)
public class ActionResolveArgs extends AbstractActionArgs {
public static final String USAGE =
"Usage: " + SliderActions.ACTION_RESOLVE
+ " "
+ ARG_PATH + " <path> "
+ "[" + ARG_LIST + "] "
+ "[" + ARG_OUTPUT + " <filename> ] "
+ "[" + ARG_DESTDIR + " <directory> ] "
;
public ActionResolveArgs() {
}
@Override
public String getActionName() {
return ACTION_RESOLVE;
}
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
@Override
public int getMinParams() {
return 0;
}
@Parameter(names = {ARG_LIST},
description = "list services")
public boolean list;
@Parameter(names = {ARG_PATH},
description = "resolve a path")
public String path;
@Parameter(names = {ARG_DESTDIR},
description = "destination directory for operations")
public File destdir;
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
description = "dest file")
public File out;
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder(ACTION_RESOLVE).append(" ");
sb.append(ARG_PATH).append(" ").append(path).append(" ");
if (list) {
sb.append(ARG_LIST).append(" ");
}
if (destdir != null) {
sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
}
if (out != null) {
sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
}
return sb.toString();
}
@Override
public void validate() throws BadCommandArgumentsException, UsageException {
super.validate();
if (StringUtils.isEmpty(path)) {
throw new BadCommandArgumentsException("Missing mandatory argument "
+ ARG_PATH);
}
if (list && out != null) {
throw new BadCommandArgumentsException("Argument "
+ ARG_OUTPUT +
" not supported for " + ARG_LIST);
}
if (out != null && destdir != null) {
throw new BadCommandArgumentsException(
ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
);
}
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
public boolean isList() {
return list;
}
public void setList(boolean list) {
this.list = list;
}
public File getDestdir() {
return destdir;
}
public void setDestdir(File destdir) {
this.destdir = destdir;
}
public File getOut() {
return out;
}
public void setOut(File out) {
this.out = out;
}
}

68
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java Normal file
View File

@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_RESOURCE},
commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE)
public class ActionResourceArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_RESOURCE;
}
@Parameter(names = {ARG_INSTALL},
description = "Install the resource(s)")
public boolean install;
@Parameter(names = {ARG_DELETE},
description = "Delete the file")
public boolean delete;
@Parameter(names = {ARG_LIST},
description = "List of installed files")
public boolean list;
@Parameter(names = {ARG_RESOURCE},
description = "Name of the file or directory")
public String resource;
@Parameter(names = {ARG_DESTDIR},
description = "The name of the folder in which to store the resources")
public String folder;
@Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)")
public boolean overwrite = false;
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 3;
}
}

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_STATUS},
commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
public class ActionStatusArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_STATUS;
}
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
description = "Output file for the status information")
public String output;
public String getOutput() {
return output;
}
public void setOutput(String output) {
this.output = output;
}
}

61
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import java.io.File;
@Parameters(commandNames = {SliderActions.ACTION_THAW},
commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
public class ActionThawArgs extends AbstractActionArgs implements
WaitTimeAccessor,
LaunchArgsAccessor {
@Override
public String getActionName() {
return SliderActions.ACTION_THAW;
}
@Override
public int getWaittime() {
return launchArgs.getWaittime();
}
@ParametersDelegate
LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
@Override
public String getRmAddress() {
return launchArgs.getRmAddress();
}
@Override
public void setWaittime(int waittime) {
launchArgs.setWaittime(waittime);
}
@Override
public File getOutputFile() {
return launchArgs.getOutputFile();
}
}

78
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java Normal file
View File

@ -0,0 +1,78 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.UsageException;
import java.io.File;
@Parameters(commandNames = {SliderActions.ACTION_TOKENS},
commandDescription = "save tokens to a file or list tokens in a file")
public class ActionTokensArgs extends AbstractActionArgs {
public static final String DUPLICATE_ARGS = "Only one of " +
ARG_SOURCE + " and " + ARG_OUTPUT + " allowed";
public static final String MISSING_KT_PROVIDER =
"Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL
+ " must be provided";
@Override
public String getActionName() {
return SliderActions.ACTION_TOKENS;
}
@Parameter(names = {ARG_OUTPUT},
description = "File to write")
public File output;
@Parameter(names = {ARG_SOURCE},
description = "source file")
public File source;
@Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
public File keytab;
@Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
public String principal="";
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public void validate() throws BadCommandArgumentsException, UsageException {
super.validate();
if (output != null && source != null) {
throw new BadCommandArgumentsException(DUPLICATE_ARGS);
}
// this is actually a !xor
if (keytab != null ^ !principal.isEmpty()) {
throw new BadCommandArgumentsException(MISSING_KT_PROVIDER);
}
}
}

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpdateArgs.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_UPDATE},
commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_UPDATE;
}
}

73
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
@Parameters(commandNames = { SliderActions.ACTION_UPGRADE },
commandDescription = SliderActions.DESCRIBE_ACTION_UPGRADE)
public class ActionUpgradeArgs extends AbstractClusterBuildingActionArgs
implements WaitTimeAccessor, LaunchArgsAccessor {
@Override
public String getActionName() {
return SliderActions.ACTION_UPGRADE;
}
@ParametersDelegate
LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
@Override
public File getOutputFile() {
return launchArgs.getOutputFile();
}
@Override
public String getRmAddress() {
return launchArgs.getRmAddress();
}
@Override
public int getWaittime() {
return launchArgs.getWaittime();
}
@Override
public void setWaittime(int waittime) {
launchArgs.setWaittime(waittime);
}
@Parameter(names={ARG_CONTAINERS}, variableArity = true,
description = "stop specific containers")
public List<String> containers = new ArrayList<>(0);
@Parameter(names={ARG_COMPONENTS}, variableArity = true,
description = "stop all containers of specific components")
public List<String> components = new ArrayList<>(0);
@Parameter(names = {ARG_FORCE},
description = "force spec upgrade operation")
public boolean force;
}

46
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameters;
/**
* The version command
*/
@Parameters(commandNames = {SliderActions.ACTION_VERSION},
commandDescription = SliderActions.DESCRIBE_ACTION_VERSION)
public class ActionVersionArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_VERSION;
}
public int getMinParams() {
return 0;
}
/**
* This action does not need hadoop services
* @return false
*/
@Override
public boolean getHadoopServicesRequired() {
return false;
}
}

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class AddonArgsDelegate extends AbstractArgsDelegate {
/**
* This is a listing of addon packages
*/
@Parameter(names = {ARG_ADDON},
arity = 2,
description = "--addon <name> <folder or package>",
splitter = DontSplitArguments.class)
public List<String> addonTuples = new ArrayList<>(0);
/**
* Get the list of addons (may be empty, but never null)
*
* @return map of named addons
*
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, String> getAddonMap() throws BadCommandArgumentsException {
return convertTupleListToMap("addon", addonTuples);
}
public List<String> getAddonTuples() {
return addonTuples;
}
}

111
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java Normal file
View File

@ -0,0 +1,111 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Delegate for application and resource options
*/
public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate {
/**
* Options key value
*/
@Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
description = ARG_OPTION + "<name> <value>",
splitter = DontSplitArguments.class)
public List<String> optionTuples = new ArrayList<>(0);
/**
* All the app component option triples
*/
@Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT, ARG_ROLEOPT}, arity = 3,
description = "Component option " + ARG_COMP_OPT +
" <component> <name> <option>",
splitter = DontSplitArguments.class)
public List<String> compOptTriples = new ArrayList<>(0);
/**
* Resource Options
*/
@Parameter(names = {ARG_RESOURCE_OPT, ARG_RESOURCE_OPT_SHORT}, arity = 2,
description = "Resource option "+ ARG_RESOURCE_OPT + "<name> <value>",
splitter = DontSplitArguments.class)
public List<String> resOptionTuples = new ArrayList<>(0);
/**
* All the resource component option triples
*/
@Parameter(names = {ARG_RES_COMP_OPT, ARG_RES_COMP_OPT_SHORT,}, arity = 3,
description = "Component resource option " + ARG_RES_COMP_OPT +
" <component> <name> <option>",
splitter = DontSplitArguments.class)
public List<String> resCompOptTriples = new ArrayList<>(0);
public Map<String, String> getOptionsMap() throws
BadCommandArgumentsException {
return convertTupleListToMap(ARG_OPTION, optionTuples);
}
/**
* Get the role heap mapping (may be empty, but never null)
* @return role heap mapping
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, Map<String, String>> getCompOptionMap() throws
BadCommandArgumentsException {
return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
}
public Map<String, String> getResourceOptionsMap() throws
BadCommandArgumentsException {
return convertTupleListToMap(ARG_RESOURCE_OPT, resOptionTuples);
}
/**
* Get the role heap mapping (may be empty, but never null)
* @return role heap mapping
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, Map<String, String>> getResourceCompOptionMap() throws
BadCommandArgumentsException {
return convertTripleListToMaps(ARG_RES_COMP_OPT, resCompOptTriples);
}
public void setOption(String key, String value) {
optionTuples.add(key);
optionTuples.add(value);
}
public void setResourceOption(String key, String value) {
resOptionTuples.add(key);
resOptionTuples.add(value);
}
}

157
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java Normal file
View File

@ -0,0 +1,157 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Static argument manipulation operations
*/
public class ArgOps {
private static final Logger
log = LoggerFactory.getLogger(ArgOps.class);
/**
* create a 3-tuple
*/
public static List<Object> triple(String msg, int min, int max) {
List<Object> l = new ArrayList<>(3);
l.add(msg);
l.add(min);
l.add(max);
return l;
}
public static void applyFileSystemBinding(String filesystemBinding,
Configuration conf) {
if (filesystemBinding != null) {
//filesystem argument was set -this overwrites any defaults in the
//configuration
FileSystem.setDefaultUri(conf, filesystemBinding);
}
}
public static void splitPairs(Collection<String> pairs,
Map<String, String> dest) {
for (String prop : pairs) {
String[] keyval = prop.split("=", 2);
if (keyval.length == 2) {
dest.put(keyval[0], keyval[1]);
}
}
}
public static void applyDefinitions(Map<String, String> definitionMap,
Configuration conf) {
for (Map.Entry<String, String> entry : definitionMap.entrySet()) {
String key = entry.getKey();
String val = entry.getValue();
log.debug("configuration[{}]<=\"{}\"", key, val);
conf.set(key, val, "command line");
}
}
/**
* Create a map from a tuple list like ['worker','2','master','1] into a map
* ['worker':'2',"master":'1'];
* Duplicate entries also trigger errors
* @param description description for errors
* @param list list to conver to tuples
* @return the map of key value pairs -unordered.
* @throws BadCommandArgumentsException odd #of arguments received
*/
public static Map<String, String> convertTupleListToMap(String description,
List<String> list) throws
BadCommandArgumentsException {
Map<String, String> results = new HashMap<>();
if (list != null && !list.isEmpty()) {
int size = list.size();
if (size % 2 != 0) {
//odd number of elements, not permitted
throw new BadCommandArgumentsException(
ErrorStrings.ERROR_PARSE_FAILURE + description);
}
for (int count = 0; count < size; count += 2) {
String key = list.get(count);
String val = list.get(count + 1);
if (results.get(key) != null) {
throw new BadCommandArgumentsException(
ErrorStrings.ERROR_DUPLICATE_ENTRY + description
+ ": " + key);
}
results.put(key, val);
}
}
return results;
}
/**
* Create a map from a tuple list like
* ['worker','heapsize','5G','master','heapsize','2M'] into a map
* ['worker':'2',"master":'1'];
* Duplicate entries also trigger errors
* @throws BadCommandArgumentsException odd #of arguments received
*/
public static Map<String, Map<String, String>> convertTripleListToMaps(String description,
List<String> list) throws BadCommandArgumentsException {
Map<String, Map<String, String>> results = new HashMap<>();
if (list != null && !list.isEmpty()) {
int size = list.size();
if (size % 3 != 0) {
//wrong number of elements, not permitted
throw new BadCommandArgumentsException(
ErrorStrings.ERROR_PARSE_FAILURE + description);
}
for (int count = 0; count < size; count += 3) {
String role = list.get(count);
String key = list.get(count + 1);
String val = list.get(count + 2);
Map<String, String> roleMap = results.get(role);
if (roleMap == null) {
//demand create new role map
roleMap = new HashMap<>();
results.put(role, roleMap);
}
if (roleMap.get(key) != null) {
throw new BadCommandArgumentsException(
ErrorStrings.ERROR_DUPLICATE_ENTRY + description
+ ": for key " + key + " under " + role);
}
roleMap.put(key, val);
}
}
return results;
}
}

162
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java Normal file
View File

@ -0,0 +1,162 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
/**
* Here are all the arguments that may be parsed by the client or server
* command lines.
*
* Important: Please keep the main list in alphabetical order
* so it is easier to see what arguments are there
*/
public interface Arguments {
String ARG_ADDON = "--addon";
String ARG_ALL = "--all";
String ARG_ALIAS = "--alias";
String ARG_APPLICATION = "--application";
String ARG_APPDEF = "--appdef";
String ARG_APP_HOME = "--apphome";
String ARG_BASE_PATH = "--basepath";
String ARG_CLIENT = "--client";
String ARG_CONFDIR = "--appconf";
String ARG_COMPONENT = "--component";
String ARG_COMPONENT_SHORT = "--comp";
String ARG_COMPONENTS = "--components";
String ARG_COMP_OPT= "--compopt";
String ARG_COMP_OPT_SHORT = "--co";
String ARG_CONFIG = "--config";
String ARG_CONTAINERS = "--containers";
String ARG_CREDENTIALS = "--credentials";
String ARG_DEBUG = "--debug";
String ARG_DEFINE = "-D";
String ARG_DELETE = "--delete";
String ARG_DEST = "--dest";
String ARG_DESTDIR = "--destdir";
String ARG_DESTFILE = "--destfile";
String ARG_EXITCODE = "--exitcode";
String ARG_FAIL = "--fail";
/**
filesystem-uri: {@value}
*/
String ARG_FILESYSTEM = "--fs";
String ARG_FILESYSTEM_LONG = "--filesystem";
String ARG_FOLDER = "--folder";
String ARG_FORCE = "--force";
String ARG_FORMAT = "--format";
String ARG_GETCERTSTORE = "--getcertstore";
String ARG_GETCONF = "--getconf";
String ARG_GETEXP = "--getexp";
String ARG_GETFILES = "--getfiles";
String ARG_HEALTHY= "--healthy";
String ARG_HELP = "--help";
String ARG_HOSTNAME = "--hostname";
String ARG_ID = "--id";
String ARG_IMAGE = "--image";
String ARG_INSTALL = "--install";
String ARG_INTERNAL = "--internal";
String ARG_KEYLEN = "--keylen";
String ARG_KEYTAB = "--keytab";
String ARG_KEYSTORE = "--keystore";
String ARG_KEYTABINSTALL = ARG_INSTALL;
String ARG_KEYTABDELETE = ARG_DELETE;
String ARG_KEYTABLIST = "--list";
String ARG_LABEL = "--label";
String ARG_LEVEL = "--level";
String ARG_LIST = "--list";
String ARG_LISTCONF = "--listconf";
String ARG_LISTEXP = "--listexp";
String ARG_LISTFILES = "--listfiles";
String ARG_LIVE = "--live";
String ARG_MANAGER = "--manager";
String ARG_MANAGER_SHORT = "--m";
String ARG_MESSAGE = "--message";
String ARG_METAINFO = "--metainfo";
String ARG_METAINFO_JSON = "--metainfojson";
String ARG_NAME = "--name";
String ARG_OPTION = "--option";
String ARG_OPTION_SHORT = "-O";
String ARG_OUTPUT = "--out";
String ARG_OUTPUT_SHORT = "-o";
String ARG_OVERWRITE = "--overwrite";
String ARG_PACKAGE = "--package";
String ARG_PASSWORD = "--password";
String ARG_PATH = "--path";
String ARG_PKGDELETE = ARG_DELETE;
String ARG_PKGINSTANCES = "--instances";
String ARG_PKGLIST = ARG_LIST;
String ARG_PRINCIPAL = "--principal";
String ARG_PROVIDER = "--provider";
String ARG_QUEUE = "--queue";
String ARG_REPLACE_PKG = "--replacepkg";
String ARG_RESOURCE = "--resource";
String ARG_RESOURCES = "--resources";
String ARG_RES_COMP_OPT = "--rescompopt";
String ARG_RES_COMP_OPT_SHORT = "--rco";
String ARG_RESOURCE_MANAGER = "--rm";
String ARG_RESOURCE_OPT = "--resopt";
String ARG_RESOURCE_OPT_SHORT = "-ro";
String ARG_SECURE = "--secure";
String ARG_SERVICETYPE = "--servicetype";
String ARG_SERVICES = "--services";
String ARG_SLIDER = "--slider";
String ARG_SOURCE = "--source";
String ARG_STATE = "--state";
String ARG_SYSPROP = "-S";
String ARG_TEMPLATE = "--template";
String ARG_TRUSTSTORE = "--truststore";
String ARG_USER = "--user";
String ARG_UPLOAD = "--upload";
String ARG_VERBOSE = "--verbose";
String ARG_VERSION = "--version";
String ARG_WAIT = "--wait";
String ARG_YARN = "--yarn";
String ARG_ZKHOSTS = "--zkhosts";
String ARG_ZKPATH = "--zkpath";
String ARG_ZKPORT = "--zkport";
/*
STOP: DO NOT ADD YOUR ARGUMENTS HERE. GO BACK AND INSERT THEM IN THE
RIGHT PLACE IN THE LIST
*/
/**
* Deprecated: use ARG_COMPONENT
*/
@Deprecated
String ARG_ROLE = "--role";
/**
* Deprecated: use ARG_COMP_OPT
*/
@Deprecated
String ARG_ROLEOPT = "--roleopt";
/**
* server: URI for the cluster
*/
String ARG_CLUSTER_URI = "-cluster-uri";
/**
* server: Path for the resource manager instance (required)
*/
String ARG_RM_ADDR = "--rm";
}

385
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java Normal file
View File

@ -0,0 +1,385 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.SliderException;
import java.util.Collection;
/**
* Slider Client CLI Args
*/
public class ClientArgs extends CommonArgs {
/*
All the arguments for specific actions
*/
/**
* This is not bonded to jcommander, it is set up
* after the construction to point to the relevant
* entry
*
* KEEP IN ALPHABETICAL ORDER
*/
private AbstractClusterBuildingActionArgs buildingActionArgs;
// =========================================================
// Keep all of these in alphabetical order. Thanks.
// =========================================================
private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs();
private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
private final ActionClientArgs actionClientArgs = new ActionClientArgs();
private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs();
private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
private final ActionDiagnosticArgs actionDiagnosticArgs = new ActionDiagnosticArgs();
private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
private final ActionInstallPackageArgs actionInstallPackageArgs = new ActionInstallPackageArgs();
private final ActionInstallKeytabArgs actionInstallKeytabArgs = new ActionInstallKeytabArgs();
private final ActionKDiagArgs actionKDiagArgs = new ActionKDiagArgs();
private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
private final ActionKillContainerArgs actionKillContainerArgs =
new ActionKillContainerArgs();
private final ActionListArgs actionListArgs = new ActionListArgs();
private final ActionLookupArgs actionLookupArgs = new ActionLookupArgs();
private final ActionNodesArgs actionNodesArgs = new ActionNodesArgs();
private final ActionPackageArgs actionPackageArgs = new ActionPackageArgs();
private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs();
private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
private final ActionThawArgs actionThawArgs = new ActionThawArgs();
private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
private final ActionUpgradeArgs actionUpgradeArgs = new ActionUpgradeArgs();
private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs();
public ClientArgs(String[] args) {
super(args);
}
public ClientArgs(Collection args) {
super(args);
}
@Override
protected void addActionArguments() {
addActions(
actionAMSuicideArgs,
actionBuildArgs,
actionClientArgs,
actionCreateArgs,
actionDependencyArgs,
actionDestroyArgs,
actionDiagnosticArgs,
actionExistsArgs,
actionFlexArgs,
actionFreezeArgs,
actionHelpArgs,
actionInstallKeytabArgs,
actionInstallPackageArgs,
actionKDiagArgs,
actionKeytabArgs,
actionKillContainerArgs,
actionListArgs,
actionLookupArgs,
actionNodesArgs,
actionPackageArgs,
actionRegistryArgs,
actionResolveArgs,
actionResourceArgs,
actionStatusArgs,
actionThawArgs,
actionTokenArgs,
actionUpdateArgs,
actionUpgradeArgs,
actionVersionArgs
);
}
@Override
public void applyDefinitions(Configuration conf) throws
BadCommandArgumentsException {
super.applyDefinitions(conf);
//RM
if (getManager() != null) {
log.debug("Setting RM to {}", getManager());
conf.set(YarnConfiguration.RM_ADDRESS, getManager());
}
if (getBasePath() != null) {
log.debug("Setting basePath to {}", getBasePath());
conf.set(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH,
getBasePath().toString());
}
}
public ActionDiagnosticArgs getActionDiagnosticArgs() {
return actionDiagnosticArgs;
}
public AbstractClusterBuildingActionArgs getBuildingActionArgs() {
return buildingActionArgs;
}
public ActionAMSuicideArgs getActionAMSuicideArgs() {
return actionAMSuicideArgs;
}
public ActionBuildArgs getActionBuildArgs() {
return actionBuildArgs;
}
public ActionInstallPackageArgs getActionInstallPackageArgs() { return actionInstallPackageArgs; }
public ActionClientArgs getActionClientArgs() { return actionClientArgs; }
public ActionPackageArgs getActionPackageArgs() { return actionPackageArgs; }
public ActionInstallKeytabArgs getActionInstallKeytabArgs() { return actionInstallKeytabArgs; }
public ActionKDiagArgs getActionKDiagArgs() {
return actionKDiagArgs;
}
public ActionKeytabArgs getActionKeytabArgs() { return actionKeytabArgs; }
public ActionUpdateArgs getActionUpdateArgs() {
return actionUpdateArgs;
}
public ActionUpgradeArgs getActionUpgradeArgs() {
return actionUpgradeArgs;
}
public ActionCreateArgs getActionCreateArgs() {
return actionCreateArgs;
}
public ActionDependencyArgs getActionDependencyArgs() {
return actionDependencyArgs;
}
public ActionDestroyArgs getActionDestroyArgs() {
return actionDestroyArgs;
}
public ActionExistsArgs getActionExistsArgs() {
return actionExistsArgs;
}
public ActionFlexArgs getActionFlexArgs() {
return actionFlexArgs;
}
public ActionFreezeArgs getActionFreezeArgs() {
return actionFreezeArgs;
}
public ActionKillContainerArgs getActionKillContainerArgs() {
return actionKillContainerArgs;
}
public ActionListArgs getActionListArgs() {
return actionListArgs;
}
public ActionNodesArgs getActionNodesArgs() {
return actionNodesArgs;
}
public ActionLookupArgs getActionLookupArgs() {
return actionLookupArgs;
}
public ActionRegistryArgs getActionRegistryArgs() {
return actionRegistryArgs;
}
public ActionResolveArgs getActionResolveArgs() {
return actionResolveArgs;
}
public ActionResourceArgs getActionResourceArgs() {
return actionResourceArgs;
}
public ActionStatusArgs getActionStatusArgs() {
return actionStatusArgs;
}
public ActionThawArgs getActionThawArgs() {
return actionThawArgs;
}
public ActionTokensArgs getActionTokenArgs() {
return actionTokenArgs;
}
/**
* Look at the chosen action and bind it as the core action for the operation.
* @throws SliderException bad argument or similar
*/
@Override
public void applyAction() throws SliderException {
String action = getAction();
if (SliderUtils.isUnset(action)) {
action = ACTION_HELP;
}
switch (action) {
case ACTION_BUILD:
bindCoreAction(actionBuildArgs);
//its a builder, so set those actions too
buildingActionArgs = actionBuildArgs;
break;
case ACTION_CREATE:
bindCoreAction(actionCreateArgs);
//its a builder, so set those actions too
buildingActionArgs = actionCreateArgs;
break;
case ACTION_FREEZE:
bindCoreAction(actionFreezeArgs);
break;
case ACTION_THAW:
bindCoreAction(actionThawArgs);
break;
case ACTION_AM_SUICIDE:
bindCoreAction(actionAMSuicideArgs);
break;
case ACTION_CLIENT:
bindCoreAction(actionClientArgs);
break;
case ACTION_DEPENDENCY:
bindCoreAction(actionDependencyArgs);
break;
case ACTION_DESTROY:
bindCoreAction(actionDestroyArgs);
break;
case ACTION_DIAGNOSTICS:
bindCoreAction(actionDiagnosticArgs);
break;
case ACTION_EXISTS:
bindCoreAction(actionExistsArgs);
break;
case ACTION_FLEX:
bindCoreAction(actionFlexArgs);
break;
case ACTION_HELP:
bindCoreAction(actionHelpArgs);
break;
case ACTION_INSTALL_KEYTAB:
bindCoreAction(actionInstallKeytabArgs);
break;
case ACTION_INSTALL_PACKAGE:
bindCoreAction(actionInstallPackageArgs);
break;
case ACTION_KDIAG:
bindCoreAction(actionKDiagArgs);
break;
case ACTION_KEYTAB:
bindCoreAction(actionKeytabArgs);
break;
case ACTION_KILL_CONTAINER:
bindCoreAction(actionKillContainerArgs);
break;
case ACTION_LIST:
bindCoreAction(actionListArgs);
break;
case ACTION_LOOKUP:
bindCoreAction(actionLookupArgs);
break;
case ACTION_NODES:
bindCoreAction(actionNodesArgs);
break;
case ACTION_PACKAGE:
bindCoreAction(actionPackageArgs);
break;
case ACTION_REGISTRY:
bindCoreAction(actionRegistryArgs);
break;
case ACTION_RESOLVE:
bindCoreAction(actionResolveArgs);
break;
case ACTION_RESOURCE:
bindCoreAction(actionResourceArgs);
break;
case ACTION_STATUS:
bindCoreAction(actionStatusArgs);
break;
case ACTION_TOKENS:
bindCoreAction(actionTokenArgs);
break;
case ACTION_UPDATE:
bindCoreAction(actionUpdateArgs);
break;
case ACTION_UPGRADE:
bindCoreAction(actionUpgradeArgs);
break;
case ACTION_VERSION:
bindCoreAction(actionVersionArgs);
break;
default:
throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
+ " " + action);
}
}
}

303
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java Normal file
View File

@ -0,0 +1,303 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterDescription;
import com.beust.jcommander.ParameterException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.UsageException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* This class contains the common argument set for all tne entry points,
* and the core parsing logic to verify that the action is on the list
* of allowed actions -and that the remaining number of arguments is
* in the range allowed
*/
public abstract class CommonArgs extends ArgOps implements SliderActions,
Arguments {
protected static final Logger log = LoggerFactory.getLogger(CommonArgs.class);
private static final int DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME = 30;
@Parameter(names = ARG_HELP, help = true)
public boolean help;
/**
-D name=value
Define an HBase configuration option which overrides any options in
the configuration XML files of the image or in the image configuration
directory. The values will be persisted.
Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
*/
public Map<String, String> definitionMap = new HashMap<String, String>();
/**
* System properties
*/
public Map<String, String> syspropsMap = new HashMap<String, String>();
/**
* fields
*/
public final JCommander commander;
private final String[] args;
private AbstractActionArgs coreAction;
/**
* get the name: relies on arg 1 being the cluster name in all operations
* @return the name argument, null if there is none
*/
public String getClusterName() {
return coreAction.getClusterName();
}
protected CommonArgs(String[] args) {
this.args = args;
commander = new JCommander(this);
}
protected CommonArgs(Collection args) {
List<String> argsAsStrings = SliderUtils.collectionToStringList(args);
this.args = argsAsStrings.toArray(new String[argsAsStrings.size()]);
commander = new JCommander(this);
}
public String usage() {
return usage(this, null);
}
public static String usage(CommonArgs serviceArgs, String commandOfInterest) {
String result = null;
StringBuilder helperMessage = new StringBuilder();
if (commandOfInterest == null) {
// JCommander.usage is too verbose for a command with many options like
// slider no short version of that is found Instead, we compose our msg by
helperMessage.append("\nUsage: slider COMMAND [options]\n");
helperMessage.append("where COMMAND is one of\n");
for (String jcommand : serviceArgs.commander.getCommands().keySet()) {
helperMessage.append(String.format("\t%-"
+ DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", jcommand,
serviceArgs.commander.getCommandDescription(jcommand) + "\n"));
}
helperMessage
.append("Most commands print help when invoked without parameters or with --help");
result = helperMessage.toString();
} else {
helperMessage.append("\nUsage: slider ").append(commandOfInterest);
helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <application>" : "");
helperMessage.append("\n");
for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
.get(commandOfInterest).getParameters()) {
String optional = paramDesc.getParameter().required() ? " (required)"
: " (optional)";
String paramName = paramDesc.getParameterized().getType() == Boolean.TYPE ? paramDesc
.getLongestName() : paramDesc.getLongestName() + " <"
+ paramDesc.getParameterized().getName() + ">";
helperMessage.append(String.format("\t%-"
+ DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", paramName,
paramDesc.getDescription() + optional + "\n"));
result = helperMessage.toString();
}
}
return result;
}
public static String usage(CommonArgs serviceArgs) {
return usage(serviceArgs, null);
}
/**
* Parse routine -includes registering the action-specific argument classes
* and postprocess it
* @throws SliderException on any problem
*/
public void parse() throws SliderException {
addActionArguments();
try {
commander.parse(getArgs());
} catch (ParameterException e) {
throw new BadCommandArgumentsException(e, "%s in %s",
e.toString(),
(getArgs() != null
? (SliderUtils.join(getArgs(),
" ", false))
: "[]"));
}
//now copy back to this class some of the attributes that are common to all
//actions
postProcess();
}
/**
* Add a command
* @param name action
* @param arg value
*/
protected void addAction(String name, Object arg) {
commander.addCommand(name, arg);
}
protected void addActions(Object... actions) {
for (Object action : actions) {
commander.addCommand(action);
}
}
/**
* Override point to add a set of actions
*/
protected void addActionArguments() {
}
/**
* validate args via {@link #validate()}
* then postprocess the arguments
*/
public void postProcess() throws SliderException {
applyAction();
validate();
//apply entry set
for (Map.Entry<String, String> entry : syspropsMap.entrySet()) {
System.setProperty(entry.getKey(), entry.getValue());
}
}
/**
* Implementors must implement their action apply routine here
*/
public abstract void applyAction() throws SliderException;
/**
* Bind the core action; this extracts any attributes that are used
* across routines
* @param action action to bind
*/
protected void bindCoreAction(AbstractActionArgs action) {
coreAction = action;
splitPairs(coreAction.definitions, definitionMap);
splitPairs(coreAction.sysprops, syspropsMap);
}
/**
* Get the core action -type depends on the action
* @return the action class
*/
public AbstractActionArgs getCoreAction() {
return coreAction;
}
/**
* Validate the arguments against the action requested
*/
public void validate() throws BadCommandArgumentsException, UsageException {
if (coreAction == null) {
throw new UsageException(ErrorStrings.ERROR_NO_ACTION + usage());
}
log.debug("action={}", getAction());
// let the action validate itself
try {
coreAction.validate();
} catch (BadCommandArgumentsException e) {
StringBuilder badArgMsgBuilder = new StringBuilder();
badArgMsgBuilder.append(e.toString()).append("\n");
badArgMsgBuilder.append(usage(this, coreAction.getActionName()));
throw new BadCommandArgumentsException(badArgMsgBuilder.toString());
} catch (UsageException e) {
StringBuilder badArgMsgBuilder = new StringBuilder();
badArgMsgBuilder.append(e.toString()).append("\n");
badArgMsgBuilder.append(usage(this, coreAction.getActionName()));
throw new UsageException(badArgMsgBuilder.toString());
}
}
/**
* Apply all the definitions on the command line to the configuration
* @param conf config
*/
public void applyDefinitions(Configuration conf) throws
BadCommandArgumentsException {
applyDefinitions(definitionMap, conf);
}
/**
* If the Filesystem binding was provided, it overrides anything in
* the configuration
* @param conf configuration
*/
public void applyFileSystemBinding(Configuration conf) {
ArgOps.applyFileSystemBinding(getFilesystemBinding(), conf);
}
public boolean isDebug() {
return coreAction.debug;
}
public String getFilesystemBinding() {
return coreAction.filesystemBinding;
}
public Path getBasePath() { return coreAction.basePath; }
public String getManager() {
return coreAction.manager;
}
public String getAction() {
return commander.getParsedCommand();
}
public List<String> getActionArgs() {
return coreAction.parameters;
}
public String[] getArgs() {
return args;
}
}

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class ComponentArgsDelegate extends AbstractArgsDelegate {
/**
* This is a listing of the roles to create
*/
@Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT, ARG_ROLE},
arity = 2,
description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3",
splitter = DontSplitArguments.class)
public List<String> componentTuples = new ArrayList<>(0);
/**
* Get the role mapping (may be empty, but never null)
* @return role mapping
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
return convertTupleListToMap("component", componentTuples);
}
public List<String> getComponentTuples() {
return componentTuples;
}
}

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.converters.IParameterSplitter;
import java.util.ArrayList;
import java.util.List;
public class DontSplitArguments implements IParameterSplitter {
@Override
public List<String> split(String value) {
List<String> list = new ArrayList<>(1);
list.add(value);
return list;
}
}

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import java.io.File;
/**
* Launch args for create and start and anything else that can start something
*/
public interface LaunchArgsAccessor extends WaitTimeAccessor {
String getRmAddress();
File getOutputFile();
}

51
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import java.io.File;
/**
* Any launch-time args
*/
public class LaunchArgsDelegate extends WaitArgsDelegate implements
LaunchArgsAccessor {
//TODO: do we need this?
@Parameter(names = ARG_RESOURCE_MANAGER,
description = "Resource manager hostname:port ",
required = false)
private String rmAddress;
@Override
public String getRmAddress() {
return rmAddress;
}
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
description = "output file for any application report")
public File outputFile;
@Override
public File getOutputFile() {
return outputFile;
}
}

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.converters.BaseConverter;
import org.apache.hadoop.fs.Path;
public class PathArgumentConverter extends BaseConverter<Path> {
public PathArgumentConverter(String optionName) {
super(optionName);
}
@Override
public Path convert(String value) {
return new Path(value);
}
}

57
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
/**
* Parameters sent by the Client to the AM
*/
public class SliderAMArgs extends CommonArgs {
SliderAMCreateAction createAction = new SliderAMCreateAction();
public SliderAMArgs(String[] args) {
super(args);
}
@Override
protected void addActionArguments() {
addActions(createAction);
}
public String getImage() {
return createAction.image;
}
/**
* This is the URI in the FS to the Slider cluster; the conf file (and any
* other cluster-specifics) can be picked up here
*/
public String getSliderClusterURI() {
return createAction.sliderClusterURI;
}
/**
* Am binding is simple: there is only one action
*/
@Override
public void applyAction() {
bindCoreAction(createAction);
}
}

74
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java Normal file
View File

@ -0,0 +1,74 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import java.io.File;
@Parameters(commandNames = {SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
public class SliderAMCreateAction extends AbstractActionArgs implements
LaunchArgsAccessor {
@Override
public String getActionName() {
return SliderActions.ACTION_CREATE;
}
@Parameter(names = ARG_IMAGE, description = "image", required = false)
public String image;
/**
* This is the URI in the FS to the Slider cluster; the conf file (and any
* other cluster-specifics) can be picked up here
*/
@Parameter(names = ARG_CLUSTER_URI,
description = "URI to the Slider cluster", required = true)
public String sliderClusterURI;
@ParametersDelegate
LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
@Override
public String getRmAddress() {
return launchArgs.getRmAddress();
}
@Override
public int getWaittime() {
return launchArgs.getWaittime();
}
@Override
public void setWaittime(int waittime) {
launchArgs.setWaittime(waittime);
}
@Override
public File getOutputFile() {
return launchArgs.getOutputFile();
}
}

113
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java Normal file
View File

@ -0,0 +1,113 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
/**
* Actions.
* Only some of these are supported by specific Slider Services; they
* are listed here to ensure the names are consistent
*/
public interface SliderActions {
String ACTION_AM_SUICIDE = "am-suicide";
String ACTION_BUILD = "build";
String ACTION_CLIENT = "client";
String ACTION_CREATE = "create";
String ACTION_DIAGNOSTICS = "diagnostics";
String ACTION_DEPENDENCY = "dependency";
String ACTION_UPDATE = "update";
String ACTION_UPGRADE = "upgrade";
String ACTION_DESTROY = "destroy";
String ACTION_ECHO = "echo";
String ACTION_EXISTS = "exists";
String ACTION_FLEX = "flex";
String ACTION_FREEZE = "stop";
String ACTION_HELP = "help";
String ACTION_INSTALL_KEYTAB = "install-keytab";
String ACTION_INSTALL_PACKAGE = "install-package";
String ACTION_KDIAG = "kdiag";
String ACTION_KEYTAB = "keytab";
String ACTION_KILL_CONTAINER = "kill-container";
String ACTION_LIST = "list";
String ACTION_LOOKUP = "lookup";
String ACTION_NODES = "nodes";
String ACTION_PACKAGE = "package";
String ACTION_PREFLIGHT = "preflight";
String ACTION_RECONFIGURE = "reconfigure";
String ACTION_REGISTRY = "registry";
String ACTION_RESOLVE = "resolve";
String ACTION_RESOURCE = "resource";
String ACTION_STATUS = "status";
String ACTION_THAW = "start";
String ACTION_TOKENS = "tokens";
String ACTION_VERSION = "version";
String DESCRIBE_ACTION_AM_SUICIDE =
"Tell the Slider Application Master to simulate a process failure by terminating itself";
String DESCRIBE_ACTION_BUILD =
"Build a Slider cluster specification, but do not start it";
String DESCRIBE_ACTION_CREATE =
"Create a live Slider application";
String DESCRIBE_ACTION_DEPENDENCY =
"Slider AM and agent dependency (libraries) management";
String DESCRIBE_ACTION_UPDATE =
"Update template for a Slider application";
String DESCRIBE_ACTION_UPGRADE =
"Rolling upgrade/downgrade the application to a newer/previous version";
String DESCRIBE_ACTION_DESTROY =
"Destroy a stopped Slider application";
String DESCRIBE_ACTION_EXISTS =
"Probe for an application running";
String DESCRIBE_ACTION_FLEX = "Flex a Slider application";
String DESCRIBE_ACTION_FREEZE =
"Stop a running application";
String DESCRIBE_ACTION_GETCONF =
"Get the configuration of an application";
String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
String DESCRIBE_ACTION_KILL_CONTAINER =
"Kill a container in the application";
String DESCRIBE_ACTION_HELP = "Print help information";
String DESCRIBE_ACTION_LIST =
"List running Slider applications";
String DESCRIBE_ACTION_LOOKUP =
"look up a YARN application";
String DESCRIBE_ACTION_NODES = "List the node information for the YARN cluster or a running application";
String DESCRIBE_ACTION_MONITOR =
"Monitor a running application";
String DESCRIBE_ACTION_REGISTRY =
"Query the registry of a YARN application";
String DESCRIBE_ACTION_RESOLVE =
"Resolve or list records in the YARN registry";
String DESCRIBE_ACTION_STATUS =
"Get the status of an application";
String DESCRIBE_ACTION_THAW =
"Start a stopped application";
String DESCRIBE_ACTION_VERSION =
"Print the Slider version information";
String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install application package." +
" Deprecated, use '" + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'.";
String DESCRIBE_ACTION_PACKAGE = "Install/list/delete application packages and list app instances that use the packages";
String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory or obtain a client keystore or truststore";
String DESCRIBE_ACTION_INSTALL_KEYTAB = "Install the Kerberos keytab." +
" Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'.";
String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client";
String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
}

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.converters.BaseConverter;
import java.net.URI;
import java.net.URISyntaxException;
public class URIArgumentConverter extends BaseConverter<URI> {
public URIArgumentConverter(String optionName) {
super(optionName);
}
@Override
public URI convert(String value) {
try {
return new URI(value);
} catch (URISyntaxException e) {
throw new RuntimeException("Cannot make a URI from " + value);
}
}
}

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.converters.BaseConverter;
import java.net.MalformedURLException;
import java.net.URL;
public class URLArgumentConverter extends BaseConverter<URL> {
public URLArgumentConverter(String optionName) {
super(optionName);
}
@Override
public URL convert(String value) {
try {
return new URL(value);
} catch (MalformedURLException e) {
throw new RuntimeException("Cannot make a URL from " + value);
}
}
}

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
public class WaitArgsDelegate extends AbstractArgsDelegate implements
WaitTimeAccessor {
//--wait [timeout]
@Parameter(names = {ARG_WAIT},
description = "time to wait for an action to complete")
public int waittime = 0;
@Override
public int getWaittime() {
return waittime;
}
@Override
public void setWaittime(int waittime) {
this.waittime = waittime;
}
}

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
public interface WaitTimeAccessor {
int getWaittime();
void setWaittime(int waittime);
}

69
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.tools;
import java.io.Serializable;
import java.util.Comparator;
/**
* Some general comparators
*/
public class Comparators {
public static class LongComparator implements Comparator<Long>, Serializable {
@Override
public int compare(Long o1, Long o2) {
long result = o1 - o2;
// need to comparisons with a diff greater than integer size
if (result < 0 ) {
return -1;
} else if (result > 0) {
return 1;
}
return 0;
}
}
public static class InvertedLongComparator implements Comparator<Long>, Serializable {
private static final LongComparator inner = new LongComparator();
@Override
public int compare(Long o1, Long o2) {
return -inner.compare(o1, o2);
}
}
/**
* Little template class to reverse any comparitor
* @param <CompareType> the type that is being compared
*/
public static class ComparatorReverser<CompareType> implements Comparator<CompareType>,
Serializable {
final Comparator<CompareType> instance;
public ComparatorReverser(Comparator<CompareType> instance) {
this.instance = instance;
}
@Override
public int compare(CompareType first, CompareType second) {
return instance.compare(second, first);
}
}
}

658
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java Normal file
View File

@ -0,0 +1,658 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.tools;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.core.exceptions.BadConfigException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* Methods to aid in config, both in the Configuration class and
* with other parts of setting up Slider-initated processes.
*
* Some of the methods take an argument of a map iterable for their sources; this allows
* the same method
*/
public class ConfigHelper {
private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
private static AtomicBoolean sliderResourceInjected =
new AtomicBoolean(false);
private static AtomicBoolean sliderResourceInjectionAttempted =
new AtomicBoolean(false);
/**
* Dump the (sorted) configuration
* @param conf config
* @return the sorted keyset
*/
public static Set<String> dumpConf(Configuration conf) {
Set<String> keys = sortedConfigKeys(conf);
for (String key : keys) {
log.info("{}={}", key, conf.get(key));
}
return keys;
}
/**
* Take a configuration and return a sorted set
* @param conf config
* @return the sorted keyset
*/
public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) {
TreeSet<String> sorted = new TreeSet<String>();
for (Map.Entry<String, String> entry : conf) {
sorted.add(entry.getKey());
}
return sorted;
}
/**
* Set an entire map full of values
*
* @param config config to patch
* @param map map of data
* @param origin origin data
*/
public static void addConfigMap(Configuration config,
Map<String, String> map,
String origin) throws BadConfigException {
addConfigMap(config, map.entrySet(), origin);
}
/**
* Set an entire map full of values
*
* @param config config to patch
* @param map map of data
* @param origin origin data
*/
public static void addConfigMap(Configuration config,
Iterable<Map.Entry<String, String>> map,
String origin) throws BadConfigException {
for (Map.Entry<String, String> mapEntry : map) {
String key = mapEntry.getKey();
String value = mapEntry.getValue();
if (value == null) {
throw new BadConfigException("Null value for property " + key);
}
config.set(key, value, origin);
}
}
/**
* Save a config file in a destination directory on a given filesystem
* @param systemConf system conf used for creating filesystems
* @param confToSave config to save
* @param confdir the directory path where the file is to go
* @param filename the filename
* @return the destination path where the file was saved
* @throws IOException IO problems
*/
public static Path saveConfig(Configuration systemConf,
Configuration confToSave,
Path confdir,
String filename) throws IOException {
FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
Path destPath = new Path(confdir, filename);
saveConfig(fs, destPath, confToSave);
return destPath;
}
/**
* Save a config
* @param fs filesystem
* @param destPath dest to save
* @param confToSave config to save
* @throws IOException IO problems
*/
public static void saveConfig(FileSystem fs,
Path destPath,
Configuration confToSave) throws
IOException {
FSDataOutputStream fos = fs.create(destPath);
try {
confToSave.writeXml(fos);
} finally {
IOUtils.closeStream(fos);
}
}
/**
* Convert to an XML string
* @param conf configuration
* @return conf
* @throws IOException
*/
public static String toXml(Configuration conf) throws IOException {
StringWriter writer = new StringWriter();
conf.writeXml(writer);
return writer.toString();
}
/**
* This will load and parse a configuration to an XML document
* @param fs filesystem
* @param path path
* @return an XML document
* @throws IOException IO failure
*/
public Document parseConfiguration(FileSystem fs,
Path path) throws
IOException {
byte[] data = loadBytes(fs, path);
//this is here to track down a parse issue
//related to configurations
String s = new String(data, 0, data.length);
log.debug("XML resource {} is \"{}\"", path, s);
/* JDK7
try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
Document document = parseConfigXML(in);
return document;
} catch (ParserConfigurationException | SAXException e) {
throw new IOException(e);
}
*/
ByteArrayInputStream in= null;
try {
in = new ByteArrayInputStream(data);
Document document = parseConfigXML(in);
return document;
} catch (ParserConfigurationException e) {
throw new IOException(e);
} catch (SAXException e) {
throw new IOException(e);
} finally {
IOUtils.closeStream(in);
}
}
public static byte[] loadBytes(FileSystem fs, Path path) throws IOException {
int len = (int) fs.getLength(path);
byte[] data = new byte[len];
/* JDK7
try(FSDataInputStream in = fs.open(path)) {
in.readFully(0, data);
}
*/
FSDataInputStream in = null;
in = fs.open(path);
try {
in.readFully(0, data);
} finally {
IOUtils.closeStream(in);
}
return data;
}
/**
* Load a configuration from ANY FS path. The normal Configuration
* loader only works with file:// URIs
* @param fs filesystem
* @param path path
* @return a loaded resource
* @throws IOException
*/
public static Configuration loadConfiguration(FileSystem fs,
Path path) throws IOException {
byte[] data = loadBytes(fs, path);
ByteArrayInputStream in2;
in2 = new ByteArrayInputStream(data);
Configuration conf1 = new Configuration(false);
conf1.addResource(in2);
//now clone it while dropping all its sources
Configuration conf2 = new Configuration(false);
String src = path.toString();
for (Map.Entry<String, String> entry : conf1) {
String key = entry.getKey();
String value = entry.getValue();
conf2.set(key, value, src);
}
return conf2;
}
/**
* Generate a config file in a destination directory on the local filesystem
* @param confdir the directory path where the file is to go
* @param filename the filename
* @return the destination path
*/
public static File saveConfig(Configuration generatingConf,
File confdir,
String filename) throws IOException {
File destPath = new File(confdir, filename);
OutputStream fos = new FileOutputStream(destPath);
try {
generatingConf.writeXml(fos);
} finally {
IOUtils.closeStream(fos);
}
return destPath;
}
/**
* Parse an XML Hadoop configuration into an XML document. x-include
* is supported, but as the location isn't passed in, relative
* URIs are out.
* @param in instream
* @return a document
* @throws ParserConfigurationException parser feature problems
* @throws IOException IO problems
* @throws SAXException XML is invalid
*/
public static Document parseConfigXML(InputStream in) throws
ParserConfigurationException,
IOException,
SAXException {
DocumentBuilderFactory docBuilderFactory
= DocumentBuilderFactory.newInstance();
//ignore all comments inside the xml file
docBuilderFactory.setIgnoringComments(true);
//allow includes in the xml file
docBuilderFactory.setNamespaceAware(true);
docBuilderFactory.setXIncludeAware(true);
DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
return builder.parse(in);
}
/**
* Load a Hadoop configuration from a local file.
* @param file file to load
* @return a configuration which hasn't actually had the load triggered
* yet.
* @throws FileNotFoundException file is not there
* @throws IOException any other IO problem
*/
public static Configuration loadConfFromFile(File file) throws
IOException {
return loadConfFromFile(file, false);
}
/**
*
* Load a Hadoop configuration from a local file.
* @param file file to load
* @param loadDefaults flag to indicate if the defaults should be loaded yet
* @return a configuration which hasn't actually had the load triggered
* yet.
* @throws FileNotFoundException file is not there
* @throws IOException any other IO problem
*/
public static Configuration loadConfFromFile(File file,
boolean loadDefaults) throws IOException {
if (!file.exists()) {
throw new FileNotFoundException("File not found :"
+ file.getAbsoluteFile());
}
Configuration conf = new Configuration(loadDefaults);
try {
conf.addResource(file.toURI().toURL());
} catch (MalformedURLException e) {
// should never happen...
throw new IOException(
"File " + file.toURI() + " doesn't have a valid URL");
}
return conf;
}
/**
* Add a configuration from a file to an existing configuration
* @param conf existing configuration
* @param file file to load
* @param overwrite flag to indicate new values should overwrite the predecessor
* @return the merged configuration
* @throws IOException
*/
public static Configuration addConfigurationFile(Configuration conf,
File file, boolean overwrite)
throws IOException {
Configuration c2 = loadConfFromFile(file, false);
mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite);
return conf;
}
/**
* Add the system env variables with the given prefix (by convention, env.)
* @param conf existing configuration
* @param prefix prefix
*/
public static void addEnvironmentVariables(Configuration conf, String prefix) {
Map<String, String> env = System.getenv();
for (Map.Entry<String, String> entry : env.entrySet()) {
conf.set(prefix + entry.getKey(),entry.getValue(), "env");
}
}
/**
* looks for the config under $confdir/$templateFilename; if not there
* loads it from /conf/templateFile.
* The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
* origin to help debug what's happening
* @param systemConf system conf
* @param confdir conf dir in FS
* @param templateFilename filename in the confdir
* @param fallbackResource resource to fall back on
* @return loaded conf
* @throws IOException IO problems
*/
public static Configuration loadTemplateConfiguration(Configuration systemConf,
Path confdir,
String templateFilename,
String fallbackResource) throws
IOException {
FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
Path templatePath = new Path(confdir, templateFilename);
return loadTemplateConfiguration(fs, templatePath, fallbackResource);
}
/**
* looks for the config under $confdir/$templateFilename; if not there
* loads it from /conf/templateFile.
* The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
* origin to help debug what's happening.
* @param fs Filesystem
* @param templatePath HDFS path for template
* @param fallbackResource resource to fall back on, or "" for no fallback
* @return loaded conf
* @throws IOException IO problems
* @throws FileNotFoundException if the path doesn't have a file and there
* was no fallback.
*/
public static Configuration loadTemplateConfiguration(FileSystem fs,
Path templatePath,
String fallbackResource)
throws IOException {
Configuration conf;
String origin;
if (fs.exists(templatePath)) {
log.debug("Loading template configuration {}", templatePath);
conf = loadConfiguration(fs, templatePath);
origin = templatePath.toString();
} else {
if (fallbackResource.isEmpty()) {
throw new FileNotFoundException("No config file found at " + templatePath);
}
log.debug("Template {} not found" +
" -reverting to classpath resource {}", templatePath, fallbackResource);
conf = new Configuration(false);
conf.addResource(fallbackResource);
origin = "Resource " + fallbackResource;
}
//force a get
conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN);
//now set the origin
conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin);
return conf;
}
/**
* For testing: dump a configuration
* @param conf configuration
* @return listing in key=value style
*/
public static String dumpConfigToString(Configuration conf) {
Set<String> sorted = sortedConfigKeys(conf);
StringBuilder builder = new StringBuilder();
for (String key : sorted) {
builder.append(key)
.append("=")
.append(conf.get(key))
.append("\n");
}
return builder.toString();
}
/**
* Merge in one configuration above another
* @param base base config
* @param merge one to merge. This MUST be a non-default-load config to avoid
* merge origin confusion
* @param origin description of the origin for the put operation
* @param overwrite flag to indicate new values should overwrite the predecessor
* @return the base with the merged values
*/
public static Configuration mergeConfigurations(Configuration base,
Iterable<Map.Entry<String, String>> merge,
String origin,
boolean overwrite) {
for (Map.Entry<String, String> entry : merge) {
String key = entry.getKey();
if (overwrite || base.get(key) == null) {
base.set(key, entry.getValue(), origin);
}
}
return base;
}
/**
* Register a resource as a default resource.
* Do not attempt to use this unless you understand that the
* order in which default resources are loaded affects the outcome,
* and that subclasses of Configuration often register new default
* resources
* @param resource the resource name
* @return the URL or null
*/
public static URL registerDefaultResource(String resource) {
URL resURL = getResourceUrl(resource);
if (resURL != null) {
Configuration.addDefaultResource(resource);
}
return resURL;
}
/**
* Load a configuration from a resource on this classpath.
* If the resource is not found, an empty configuration is returned
* @param resource the resource name
* @return the loaded configuration.
*/
public static Configuration loadFromResource(String resource) {
Configuration conf = new Configuration(false);
URL resURL = getResourceUrl(resource);
if (resURL != null) {
log.debug("loaded resources from {}", resURL);
conf.addResource(resource);
} else{
log.debug("failed to find {} on the classpath", resource);
}
return conf;
}
/**
* Get the URL to a resource, null if not on the CP
* @param resource resource to look for
* @return the URL or null
*/
public static URL getResourceUrl(String resource) {
return ConfigHelper.class.getClassLoader()
.getResource(resource);
}
/**
* Load a resource that must be on the classpath
* @param resource the resource name
* @return the loaded configuration
* @throws FileNotFoundException if the resource is missing
*/
public static Configuration loadMandatoryResource(String resource)
throws FileNotFoundException {
Configuration conf = new Configuration(false);
URL resURL = getResourceUrl(resource);
if (resURL != null) {
log.debug("loaded resources from {}", resURL);
conf.addResource(resource);
} else {
throw new FileNotFoundException(resource);
}
return conf;
}
/**
* Propagate a property from a source to a dest config, with a best-effort
* attempt at propagating the origin.
* If the
* @param dest destination
* @param src source
* @param key key to try to copy
* @return true if the key was found and propagated
*/
public static boolean propagate(Configuration dest,
Configuration src,
String key) {
String val = src.get(key);
if (val != null) {
String[] origin = src.getPropertySources(key);
if (origin != null && origin.length > 0) {
dest.set(key, val, origin[0]);
} else {
dest.set(key, val);
return true;
}
}
return false;
}
/**
* Take a configuration, return a hash map
* @param conf conf
* @return hash map
*/
public static Map<String, String> buildMapFromConfiguration(Configuration conf) {
Map<String, String> map = new HashMap<String, String>();
return SliderUtils.mergeEntries(map, conf);
}
/**
* This goes through the keyset of one configuration and retrieves each value
* from a value source -a different or the same configuration. This triggers
* the property resolution process of the value, resolving any variables against
* in-config or inherited configurations
* @param keysource source of keys
* @param valuesource the source of values
* @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
*/
public static Configuration resolveConfiguration(
Iterable<Map.Entry<String, String>> keysource,
Configuration valuesource) {
Configuration result = new Configuration(false);
for (Map.Entry<String, String> entry : keysource) {
String key = entry.getKey();
String value = valuesource.get(key);
Preconditions.checkState(value != null,
"no reference for \"%s\" in values", key);
result.set(key, value);
}
return result;
}
/**
* Register anything we consider deprecated
*/
public static void registerDeprecatedConfigItems() {
Configuration.addDeprecation(
SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
Configuration.addDeprecation(
SliderXmlConfKeys.REGISTRY_PATH,
RegistryConstants.KEY_REGISTRY_ZK_ROOT);
}
/**
* Load a configuration with the {@link SliderKeys#SLIDER_XML} resource
* included
* @return a configuration instance
*/
public static Configuration loadSliderConfiguration() {
Configuration conf = new Configuration();
conf.addResource(SliderKeys.SLIDER_XML);
return conf;
}
/**
* Inject the {@link SliderKeys#SLIDER_XML} resource
* into the configuration resources <i>of all configurations</i>.
* <p>
* This operation is idempotent.
* <p>
* If the resource is not on the classpath, downgrades, rather than
* fails.
* @return true if the resource was found and loaded.
*/
public static synchronized boolean injectSliderXMLResource() {
if (sliderResourceInjectionAttempted.getAndSet(true)) {
return sliderResourceInjected.get();
}
URL resourceUrl = getResourceUrl(SliderKeys.SLIDER_XML);
if (resourceUrl != null) {
Configuration.addDefaultResource(SliderKeys.SLIDER_XML);
sliderResourceInjected.set(true);
}
return sliderResourceInjected.get();
}
}

915
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java Normal file
View File

@ -0,0 +1,915 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.tools;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
import org.apache.slider.core.persist.Filenames;
import org.apache.slider.core.persist.InstancePaths;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FilenameFilter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import static org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
public class CoreFileSystem {
private static final Logger
log = LoggerFactory.getLogger(CoreFileSystem.class);
private static final String UTF_8 = "UTF-8";
protected final FileSystem fileSystem;
protected final Configuration configuration;
public CoreFileSystem(FileSystem fileSystem, Configuration configuration) {
Preconditions.checkNotNull(fileSystem,
"Cannot create a CoreFileSystem with a null FileSystem");
Preconditions.checkNotNull(configuration,
"Cannot create a CoreFileSystem with a null Configuration");
this.fileSystem = fileSystem;
this.configuration = configuration;
}
public CoreFileSystem(Configuration configuration) throws IOException {
Preconditions.checkNotNull(configuration,
"Cannot create a CoreFileSystem with a null Configuration");
this.fileSystem = FileSystem.get(configuration);
this.configuration = fileSystem.getConf();
}
/**
* Get the temp path for this cluster
* @param clustername name of the cluster
* @return path for temp files (is not purged)
*/
public Path getTempPathForCluster(String clustername) {
Path clusterDir = buildClusterDirPath(clustername);
return new Path(clusterDir, SliderKeys.TMP_DIR_PREFIX);
}
/**
* Returns the underlying FileSystem for this object.
*
* @return filesystem
*/
public FileSystem getFileSystem() {
return fileSystem;
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("CoreFileSystem{");
sb.append("fileSystem=").append(fileSystem.getUri());
sb.append('}');
return sb.toString();
}
/**
* Build up the path string for a cluster instance -no attempt to
* create the directory is made
*
* @param clustername name of the cluster
* @return the path for persistent data
*/
public Path buildClusterDirPath(String clustername) {
Preconditions.checkNotNull(clustername);
Path path = getBaseApplicationPath();
return new Path(path, SliderKeys.CLUSTER_DIRECTORY + "/" + clustername);
}
/**
* Build up the path string for app def folder -no attempt to
* create the directory is made
*
* @param clustername name of the cluster
* @return the path for persistent data
*/
public Path buildAppDefDirPath(String clustername) {
Path path = buildClusterDirPath(clustername);
return new Path(path, SliderKeys.APP_DEF_DIR);
}
/**
* Build up the path string for addon folder -no attempt to
* create the directory is made
*
* @param clustername name of the cluster
* @return the path for persistent data
*/
public Path buildAddonDirPath(String clustername, String addonId) {
Preconditions.checkNotNull(addonId);
Path path = buildClusterDirPath(clustername);
return new Path(path, SliderKeys.ADDONS_DIR + "/" + addonId);
}
/**
* Build up the path string for package install location -no attempt to
* create the directory is made
*
* @return the path for persistent app package
*/
public Path buildPackageDirPath(String packageName, String packageVersion) {
Preconditions.checkNotNull(packageName);
Path path = getBaseApplicationPath();
path = new Path(path, SliderKeys.PACKAGE_DIRECTORY + "/" + packageName);
if (SliderUtils.isSet(packageVersion)) {
path = new Path(path, packageVersion);
}
return path;
}
/**
* Build up the path string for package install location -no attempt to
* create the directory is made
*
* @return the path for persistent app package
*/
public Path buildClusterSecurityDirPath(String clusterName) {
Preconditions.checkNotNull(clusterName);
Path path = buildClusterDirPath(clusterName);
return new Path(path, SliderKeys.SECURITY_DIR);
}
/**
* Build up the path string for keytab install location -no attempt to
* create the directory is made
*
* @return the path for keytab
*/
public Path buildKeytabInstallationDirPath(String keytabFolder) {
Preconditions.checkNotNull(keytabFolder);
Path path = getBaseApplicationPath();
return new Path(path, SliderKeys.KEYTAB_DIR + "/" + keytabFolder);
}
/**
* Build up the path string for keytab install location -no attempt to
* create the directory is made
*
* @return the path for keytab installation location
*/
public Path buildKeytabPath(String keytabDir, String keytabName, String clusterName) {
Path homePath = getHomeDirectory();
Path baseKeytabDir;
if (keytabDir != null) {
baseKeytabDir = new Path(homePath, keytabDir);
} else {
baseKeytabDir = new Path(buildClusterDirPath(clusterName),
SliderKeys.KEYTAB_DIR);
}
return keytabName == null ? baseKeytabDir :
new Path(baseKeytabDir, keytabName);
}
/**
* Build up the path string for resource install location -no attempt to
* create the directory is made
*
* @return the path for resource
*/
public Path buildResourcePath(String resourceFolder) {
Preconditions.checkNotNull(resourceFolder);
Path path = getBaseApplicationPath();
return new Path(path, SliderKeys.RESOURCE_DIR + "/" + resourceFolder);
}
/**
* Build up the path string for resource install location -no attempt to
* create the directory is made
*
* @return the path for resource
*/
public Path buildResourcePath(String dirName, String fileName) {
Preconditions.checkNotNull(dirName);
Preconditions.checkNotNull(fileName);
Path path = getBaseApplicationPath();
return new Path(path, SliderKeys.RESOURCE_DIR + "/" + dirName + "/" + fileName);
}
/**
* Build up the path string for cluster resource install location -no
* attempt to create the directory is made
*
* @return the path for resource
*/
public Path buildClusterResourcePath(String clusterName, String component) {
Preconditions.checkNotNull(clusterName);
Path path = buildClusterDirPath(clusterName);
return new Path(path, SliderKeys.RESOURCE_DIR + "/" + component);
}
/**
* Build up the path string for cluster resource install location -no
* attempt to create the directory is made
*
* @return the path for resource
*/
public Path buildClusterResourcePath(String clusterName) {
Preconditions.checkNotNull(clusterName);
Path path = buildClusterDirPath(clusterName);
return new Path(path, SliderKeys.RESOURCE_DIR);
}
/**
* Create the Slider cluster path for a named cluster and all its subdirs
* This is a directory; a mkdirs() operation is executed
* to ensure that it is there.
*
* @param clustername name of the cluster
* @return the path to the cluster directory
* @throws java.io.IOException trouble
* @throws SliderException slider-specific exceptions
*/
public Path createClusterDirectories(String clustername, Configuration conf)
throws IOException, SliderException {
Path clusterDirectory = buildClusterDirPath(clustername);
InstancePaths instancePaths = new InstancePaths(clusterDirectory);
createClusterDirectories(instancePaths);
return clusterDirectory;
}
/**
* Create the Slider cluster path for a named cluster and all its subdirs
* This is a directory; a mkdirs() operation is executed
* to ensure that it is there.
*
* @param instancePaths instance paths
* @throws IOException trouble
* @throws SliderException slider-specific exceptions
*/
public void createClusterDirectories(InstancePaths instancePaths) throws
IOException, SliderException {
Path instanceDir = instancePaths.instanceDir;
verifyDirectoryNonexistent(instanceDir);
FsPermission clusterPerms = getInstanceDirectoryPermissions();
createWithPermissions(instanceDir, clusterPerms);
createWithPermissions(instancePaths.snapshotConfPath, clusterPerms);
createWithPermissions(instancePaths.generatedConfPath, clusterPerms);
createWithPermissions(instancePaths.historyPath, clusterPerms);
createWithPermissions(instancePaths.tmpPathAM, clusterPerms);
// Data Directory
String dataOpts =
configuration.get(SliderXmlConfKeys.DATA_DIRECTORY_PERMISSIONS,
SliderXmlConfKeys.DEFAULT_DATA_DIRECTORY_PERMISSIONS);
log.debug("Setting data directory permissions to {}", dataOpts);
createWithPermissions(instancePaths.dataPath, new FsPermission(dataOpts));
}
/**
* Create a directory with the given permissions.
*
* @param dir directory
* @param clusterPerms cluster permissions
* @throws IOException IO problem
* @throws BadClusterStateException any cluster state problem
*/
public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
IOException,
BadClusterStateException {
if (fileSystem.isFile(dir)) {
// HADOOP-9361 shows some filesystems don't correctly fail here
throw new BadClusterStateException(
"Cannot create a directory over a file %s", dir);
}
log.debug("mkdir {} with perms {}", dir, clusterPerms);
//no mask whatoever
fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
fileSystem.mkdirs(dir, clusterPerms);
//and force set it anyway just to make sure
fileSystem.setPermission(dir, clusterPerms);
}
/**
* Get the permissions of a path
*
* @param path path to check
* @return the permissions
* @throws IOException any IO problem (including file not found)
*/
public FsPermission getPathPermissions(Path path) throws IOException {
FileStatus status = fileSystem.getFileStatus(path);
return status.getPermission();
}
public FsPermission getInstanceDirectoryPermissions() {
String clusterDirPermsOct =
configuration.get(CLUSTER_DIRECTORY_PERMISSIONS,
DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
return new FsPermission(clusterDirPermsOct);
}
/**
* Verify that the cluster directory is not present
*
* @param clustername name of the cluster
* @param clusterDirectory actual directory to look for
* @throws IOException trouble with FS
* @throws SliderException If the directory exists
*/
public void verifyClusterDirectoryNonexistent(String clustername,
Path clusterDirectory)
throws IOException, SliderException {
if (fileSystem.exists(clusterDirectory)) {
throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
ErrorStrings.PRINTF_E_INSTANCE_ALREADY_EXISTS, clustername,
clusterDirectory);
}
}
/**
* Verify that the given directory is not present
*
* @param clusterDirectory actual directory to look for
* @throws IOException trouble with FS
* @throws SliderException If the directory exists
*/
public void verifyDirectoryNonexistent(Path clusterDirectory) throws
IOException,
SliderException {
if (fileSystem.exists(clusterDirectory)) {
log.error("Dir {} exists: {}",
clusterDirectory,
listFSDir(clusterDirectory));
throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
ErrorStrings.PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS,
clusterDirectory);
}
}
/**
* Verify that a user has write access to a directory.
* It does this by creating then deleting a temp file
*
* @param dirPath actual directory to look for
* @throws FileNotFoundException file not found
* @throws IOException trouble with FS
* @throws BadClusterStateException if the directory is not writeable
*/
public void verifyDirectoryWriteAccess(Path dirPath) throws IOException,
SliderException {
verifyPathExists(dirPath);
Path tempFile = new Path(dirPath, "tmp-file-for-checks");
try {
FSDataOutputStream out ;
out = fileSystem.create(tempFile, true);
IOUtils.closeStream(out);
fileSystem.delete(tempFile, false);
} catch (IOException e) {
log.warn("Failed to create file {}: {}", tempFile, e);
throw new BadClusterStateException(e,
"Unable to write to directory %s : %s", dirPath, e.toString());
}
}
/**
* Verify that a path exists
* @param path path to check
* @throws FileNotFoundException file not found
* @throws IOException trouble with FS
*/
public void verifyPathExists(Path path) throws IOException {
if (!fileSystem.exists(path)) {
throw new FileNotFoundException(path.toString());
}
}
/**
* Verify that a path exists
* @param path path to check
* @throws FileNotFoundException file not found or is not a file
* @throws IOException trouble with FS
*/
public void verifyFileExists(Path path) throws IOException {
FileStatus status = fileSystem.getFileStatus(path);
if (!status.isFile()) {
throw new FileNotFoundException("Not a file: " + path.toString());
}
}
/**
* Given a path, check if it exists and is a file
*
* @param path
* absolute path to the file to check
* @returns true if and only if path exists and is a file, false for all other
* reasons including if file check throws IOException
*/
public boolean isFile(Path path) {
boolean isFile = false;
try {
FileStatus status = fileSystem.getFileStatus(path);
if (status.isFile()) {
isFile = true;
}
} catch (IOException e) {
// ignore, isFile is already set to false
}
return isFile;
}
/**
* Verify that a file exists in the zip file given by path
* @param path path to zip file
* @param file file expected to be in zip
* @throws FileNotFoundException file not found or is not a zip file
* @throws IOException trouble with FS
*/
public void verifyFileExistsInZip(Path path, String file) throws IOException {
fileSystem.copyToLocalFile(path, new Path("/tmp"));
File dst = new File((new Path("/tmp", path.getName())).toString());
Enumeration<? extends ZipEntry> entries;
ZipFile zipFile = new ZipFile(dst);
boolean found = false;
try {
entries = zipFile.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
String nm = entry.getName();
if (nm.endsWith(file)) {
found = true;
break;
}
}
} finally {
zipFile.close();
}
dst.delete();
if (!found) throw new FileNotFoundException("file: " + file + " not found in " + path);
log.info("Verification of " + path + " passed");
}
/**
* Create the application-instance specific temporary directory
* in the DFS
*
* @param clustername name of the cluster
* @param subdir application ID
* @return the path; this directory will already have been created
*/
public Path createAppInstanceTempPath(String clustername, String subdir)
throws IOException {
Path tmp = getTempPathForCluster(clustername);
Path instancePath = new Path(tmp, subdir);
fileSystem.mkdirs(instancePath);
return instancePath;
}
/**
* Create the application-instance specific temporary directory
* in the DFS
*
* @param clustername name of the cluster
* @return the path; this directory will already have been deleted
*/
public Path purgeAppInstanceTempFiles(String clustername) throws
IOException {
Path tmp = getTempPathForCluster(clustername);
fileSystem.delete(tmp, true);
return tmp;
}
/**
* Get the base path
*
* @return the base path optionally configured by
* {@link SliderXmlConfKeys#KEY_SLIDER_BASE_PATH}
*/
public Path getBaseApplicationPath() {
String configuredBasePath = configuration.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH);
return configuredBasePath != null ? new Path(configuredBasePath) :
new Path(getHomeDirectory(), SliderKeys.SLIDER_BASE_DIRECTORY);
}
/**
* Get slider dependency parent dir in HDFS
*
* @return the parent dir path of slider.tar.gz in HDFS
*/
public Path getDependencyPath() {
String parentDir = (SliderUtils.isHdp()) ? SliderKeys.SLIDER_DEPENDENCY_HDP_PARENT_DIR
+ SliderKeys.SLIDER_DEPENDENCY_DIR
: SliderKeys.SLIDER_DEPENDENCY_DIR;
Path dependencyPath = new Path(String.format(parentDir,
SliderUtils.getSliderVersion()));
return dependencyPath;
}
/**
* Get slider.tar.gz absolute filepath in HDFS
*
* @return the absolute path to slider.tar.gz in HDFS
*/
public Path getDependencyTarGzip() {
Path dependencyLibAmPath = getDependencyPath();
Path dependencyLibTarGzip = new Path(
dependencyLibAmPath.toUri().toString(),
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME
+ SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
return dependencyLibTarGzip;
}
public Path getHomeDirectory() {
return fileSystem.getHomeDirectory();
}
public boolean maybeAddImagePath(Map<String, LocalResource> localResources,
Path imagePath) throws IOException {
if (imagePath != null) {
LocalResource resource = createAmResource(imagePath,
LocalResourceType.ARCHIVE);
localResources.put(SliderKeys.LOCAL_TARBALL_INSTALL_SUBDIR, resource);
return true;
} else {
return false;
}
}
public boolean maybeAddImagePath(Map<String, LocalResource> localResources,
String imagePath) throws IOException {
return imagePath != null &&
maybeAddImagePath(localResources, new Path(imagePath));
}
/**
* Create an AM resource from the
*
* @param destPath dest path in filesystem
* @param resourceType resource type
* @return the resource set up wih application-level visibility and the
* timestamp & size set from the file stats.
*/
public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException {
FileStatus destStatus = fileSystem.getFileStatus(destPath);
LocalResource amResource = Records.newRecord(LocalResource.class);
amResource.setType(resourceType);
// Set visibility of the resource
// Setting to most private option
amResource.setVisibility(LocalResourceVisibility.APPLICATION);
// Set the resource to be copied over
amResource.setResource(ConverterUtils.getYarnUrlFromPath(fileSystem
.resolvePath(destStatus.getPath())));
// Set timestamp and length of file so that the framework
// can do basic sanity checks for the local resource
// after it has been copied over to ensure it is the same
// resource the client intended to use with the application
amResource.setTimestamp(destStatus.getModificationTime());
amResource.setSize(destStatus.getLen());
return amResource;
}
/**
* Register all files under a fs path as a directory to push out
*
* @param srcDir src dir
* @param destRelativeDir dest dir (no trailing /)
* @return the map of entries
*/
public Map<String, LocalResource> submitDirectory(Path srcDir, String destRelativeDir) throws IOException {
//now register each of the files in the directory to be
//copied to the destination
FileStatus[] fileset = fileSystem.listStatus(srcDir);
Map<String, LocalResource> localResources =
new HashMap<String, LocalResource>(fileset.length);
for (FileStatus entry : fileset) {
LocalResource resource = createAmResource(entry.getPath(),
LocalResourceType.FILE);
String relativePath = destRelativeDir + "/" + entry.getPath().getName();
localResources.put(relativePath, resource);
}
return localResources;
}
/**
* Submit a JAR containing a specific class, returning
* the resource to be mapped in
*
* @param clazz class to look for
* @param subdir subdirectory (expected to end in a "/")
* @param jarName <i>At the destination</i>
* @return the local resource ref
* @throws IOException trouble copying to HDFS
*/
public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
throws IOException, SliderException {
File localFile = SliderUtils.findContainingJarOrFail(clazz);
return submitFile(localFile, tempPath, subdir, jarName);
}
/**
* Submit a local file to the filesystem references by the instance's cluster
* filesystem
*
* @param localFile filename
* @param subdir subdirectory (expected to end in a "/")
* @param destFileName destination filename
* @return the local resource ref
* @throws IOException trouble copying to HDFS
*/
public LocalResource submitFile(File localFile, Path tempPath, String subdir, String destFileName)
throws IOException {
Path src = new Path(localFile.toString());
Path subdirPath = new Path(tempPath, subdir);
fileSystem.mkdirs(subdirPath);
Path destPath = new Path(subdirPath, destFileName);
log.debug("Copying {} (size={} bytes) to {}", localFile, localFile.length(), destPath);
fileSystem.copyFromLocalFile(false, true, src, destPath);
// Set the type of resource - file or archive
// archives are untarred at destination
// we don't need the jar file to be untarred for now
return createAmResource(destPath, LocalResourceType.FILE);
}
/**
* Submit the AM tar.gz resource referenced by the instance's cluster
* filesystem. Also, update the providerResources object with the new
* resource.
*
* @param providerResources
* the provider resource map to be updated
* @throws IOException
* trouble copying to HDFS
*/
public void submitTarGzipAndUpdate(
Map<String, LocalResource> providerResources) throws IOException,
BadClusterStateException {
Path dependencyLibTarGzip = getDependencyTarGzip();
LocalResource lc = createAmResource(dependencyLibTarGzip,
LocalResourceType.ARCHIVE);
providerResources.put(SliderKeys.SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK, lc);
}
/**
* Copy local file(s) to destination HDFS directory. If {@code localPath} is a
* local directory then all files matching the {@code filenameFilter}
* (optional) are copied, otherwise {@code filenameFilter} is ignored.
*
* @param localPath
* a local file or directory path
* @param filenameFilter
* if {@code localPath} is a directory then filenameFilter is used as
* a filter (if specified)
* @param destDir
* the destination HDFS directory where the file(s) should be copied
* @param fp
* file permissions of all the directories and files that will be
* created in this api
* @throws IOException
*/
public void copyLocalFilesToHdfs(File localPath,
FilenameFilter filenameFilter, Path destDir, FsPermission fp)
throws IOException {
if (localPath == null || destDir == null) {
throw new IOException("Either localPath or destDir is null");
}
fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
"000");
fileSystem.mkdirs(destDir, fp);
if (localPath.isDirectory()) {
// copy all local files under localPath to destDir (honoring filename
// filter if provided
File[] localFiles = localPath.listFiles(filenameFilter);
Path[] localFilePaths = new Path[localFiles.length];
int i = 0;
for (File localFile : localFiles) {
localFilePaths[i++] = new Path(localFile.getPath());
}
log.info("Copying {} files from {} to {}", i, localPath.toURI(),
destDir.toUri());
fileSystem.copyFromLocalFile(false, true, localFilePaths, destDir);
} else {
log.info("Copying file {} to {}", localPath.toURI(), destDir.toUri());
fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
destDir);
}
// set permissions for all the files created in the destDir
fileSystem.setPermission(destDir, fp);
}
public void copyLocalFileToHdfs(File localPath,
Path destPath, FsPermission fp)
throws IOException {
if (localPath == null || destPath == null) {
throw new IOException("Either localPath or destPath is null");
}
fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
"000");
fileSystem.mkdirs(destPath.getParent(), fp);
log.info("Copying file {} to {}", localPath.toURI(), destPath.toUri());
fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
destPath);
// set file permissions of the destPath
fileSystem.setPermission(destPath, fp);
}
public void copyHdfsFileToLocal(Path hdfsPath, File destFile)
throws IOException {
if (hdfsPath == null || destFile == null) {
throw new IOException("Either hdfsPath or destPath is null");
}
log.info("Copying file {} to {}", hdfsPath.toUri(), destFile.toURI());
Path destPath = new Path(destFile.getPath());
fileSystem.copyToLocalFile(hdfsPath, destPath);
}
/**
* list entries in a filesystem directory
*
* @param path directory
* @return a listing, one to a line
* @throws IOException
*/
public String listFSDir(Path path) throws IOException {
FileStatus[] stats = fileSystem.listStatus(path);
StringBuilder builder = new StringBuilder();
for (FileStatus stat : stats) {
builder.append(stat.getPath().toString())
.append("\t")
.append(stat.getLen())
.append("\n");
}
return builder.toString();
}
/**
* List all application instances persisted for this user, giving the
* path. The instance name is the last element in the path
* @return a possibly empty map of application instance names to paths
*/
public Map<String, Path> listPersistentInstances() throws IOException {
FileSystem fs = getFileSystem();
Path path = new Path(getBaseApplicationPath(), SliderKeys.CLUSTER_DIRECTORY);
log.debug("Looking for all persisted application at {}", path.toString());
if (!fs.exists(path)) {
// special case: no instances have ever been created
return new HashMap<String, Path>(0);
}
FileStatus[] statuses = fs.listStatus(path);
Map<String, Path> instances = new HashMap<String, Path>(statuses.length);
// enum the child entries
for (FileStatus status : statuses) {
if (status.isDirectory()) {
// for directories, look for an internal.json underneath
Path child = status.getPath();
Path internalJson = new Path(child, Filenames.INTERNAL);
if (fs.exists(internalJson)) {
// success => this is an instance
instances.put(child.getName(), child);
} else {
log.info("Malformed cluster found at {}. It does not appear to be a valid persisted instance.",
child.toString());
}
}
}
return instances;
}
public void touch(Path path, boolean overwrite) throws IOException {
FSDataOutputStream out = null;
try {
out = fileSystem.create(path, overwrite);
} finally {
IOUtils.closeStream(out);
}
}
public void cat(Path path, boolean overwrite, String data) throws IOException {
FSDataOutputStream out = null;
try {
out = fileSystem.create(path, overwrite);
byte[] bytes = data.getBytes(Charset.forName("UTF-8"));
out.write(bytes);
} finally {
IOUtils.closeStream(out);
}
}
public String cat(Path path) throws IOException {
FileStatus status = fileSystem.getFileStatus(path);
byte[] b = new byte[(int) status.getLen()];
FSDataInputStream in = null;
try {
in = fileSystem.open(path);
int count = in.read(b);
return new String(b, 0, count, UTF_8);
} finally {
IOUtils.closeStream(in);
}
}
/**
* Create a path that must exist in the cluster fs
* @param uri uri to create
* @return the path
* @throws SliderException if the path does not exist
*/
public Path createPathThatMustExist(String uri) throws
SliderException, IOException {
Preconditions.checkNotNull(uri);
Path path = new Path(uri);
verifyPathExists(path);
return path;
}
/**
* Locate an application conf json in the FS. This includes a check to verify
* that the file is there.
*
* @param clustername name of the cluster
* @return the path to the spec.
* @throws IOException IO problems
* @throws SliderException if the path isn't there
*/
public Path locateInstanceDefinition(String clustername) throws IOException,
SliderException {
Path clusterDirectory = buildClusterDirPath(clustername);
Path appConfPath =
new Path(clusterDirectory, Filenames.APPCONF);
verifyClusterSpecExists(clustername, appConfPath);
return appConfPath;
}
/**
* Verify that a cluster specification exists
* @param clustername name of the cluster (For errors only)
* @param clusterSpecPath cluster specification path
* @throws IOException IO problems
* @throws SliderException if the cluster specification is not present
*/
public void verifyClusterSpecExists(String clustername, Path clusterSpecPath)
throws IOException,
SliderException {
if (!fileSystem.isFile(clusterSpecPath)) {
log.debug("Missing specification file {}", clusterSpecPath);
throw UnknownApplicationInstanceException.unknownInstance(
clustername + "\n (definition not found at " + clusterSpecPath);
}
}
}

Some files were not shown because too many files have changed in this diff Show More