From fecf22b2fd5321d13f0c79d9daf27fbe1f84c9cb Mon Sep 17 00:00:00 2001 From: Jian He Date: Tue, 29 Aug 2017 11:09:00 -0700 Subject: [PATCH] YARN-7113. Clean up packaging and dependencies for yarn-native-services. Contributed by Billie Rinaldi --- NOTICE.txt | 14 + .../resources/assemblies/hadoop-yarn-dist.xml | 8 - .../assemblies/hadoop-yarn-services-api.xml | 36 - .../assemblies/hadoop-yarn-services-dist.xml | 30 - hadoop-project/pom.xml | 19 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 35 +- .../hadoop-yarn-services-api/pom.xml | 104 +- .../yarn/service/webapp/ApiServerWebApp.java | 4 +- .../{services-rest-api => api-server}/app | 0 .../hadoop-yarn-services-core/pom.xml | 243 +--- .../client/params/ActionKDiagArgs.java | 76 -- .../service/client/params/ClientArgs.java | 5 - .../YarnRegistryViewForProviders.java | 8 +- .../yarn/service/utils/KerberosDiags.java | 680 ----------- .../yarn/service/utils/SliderUtils.java | 1088 ----------------- .../hadoop/yarn/service/ServiceTestUtils.java | 28 + .../yarn/service/TestServiceApiUtil.java | 38 +- .../yarn/service/TestYarnNativeServices.java | 10 +- .../yarn/service/client/TestServiceCLI.java | 1 - .../yarn/service/conf/TestAppJsonResolve.java | 30 +- .../service/conf/TestLoadExampleAppJson.java | 11 +- .../providers/TestAbstractClientProvider.java | 10 +- hadoop-yarn-project/pom.xml | 4 + 23 files changed, 174 insertions(+), 2308 deletions(-) delete mode 100644 hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml delete mode 100644 hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/{services-rest-api => api-server}/app (100%) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java diff --git a/NOTICE.txt b/NOTICE.txt index 0718909cb1..f3af2f7030 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -581,3 +581,17 @@ The binary distribution of this product bundles binaries of Ehcache 3.3.1, which has the following notices: * Ehcache V3 Copyright 2014-2016 Terracotta, Inc. + +JCommander (https://github.com/cbeust/jcommander), +which has the following notices: + * Copyright 2010 Cedric Beust cedric@beust.com + +The binary distribution of this product bundles binaries of +snakeyaml (https://bitbucket.org/asomov/snakeyaml), +which has the following notices: + * Copyright (c) 2008, http://www.snakeyaml.org + +The binary distribution of this product bundles binaries of +swagger-annotations (https://github.com/swagger-api/swagger-core), +which has the following notices: + * Copyright 2016 SmartBear Software diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 8aeeabd2bf..8b3d2926fb 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -97,10 +97,6 @@ hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf etc/hadoop - - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target/hadoop-yarn-services-core-${project.version} - /share/hadoop/${hadoop.component}/lib/services - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target /share/hadoop/${hadoop.component}/sources @@ -108,10 +104,6 @@ *-sources.jar - - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version} - /share/hadoop/${hadoop.component}/lib/services-api - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target /share/hadoop/${hadoop.component}/sources diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml deleted file mode 100644 index 589f724848..0000000000 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - hadoop-yarn-services-api-dist - - dir - - false - - - false - - com.fasterxml.jackson.jaxrs:jackson-jaxrs-base - com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider - com.fasterxml.jackson.module:jackson-module-jaxb-annotations - io.swagger:swagger-annotations - - - - diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml deleted file mode 100644 index 1b81f98be5..0000000000 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - hadoop-yarn-services-dist - - dir - - false - - - false - - - diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 3923183c82..3cb956821c 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -145,7 +145,8 @@ ${project.version} 1.5.4 - 1.4 + 1.16 + 1.30 @@ -607,6 +608,11 @@ javax.servlet-api 3.1.0 + + javax.ws.rs + jsr311-api + 1.1.1 + org.eclipse.jetty jetty-server @@ -1343,9 +1349,14 @@ ${jackson2.version} - org.apache.maven.doxia - doxia-module-markdown - ${maven-doxia-module-markdown.version} + org.yaml + snakeyaml + ${snakeyaml.version} + + + com.beust + jcommander + ${jcommander.version} diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 0a534555a6..1db585e6e1 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -31,6 +31,7 @@ function hadoop_usage hadoop_add_option "--hosts filename" "list of hosts to use in worker mode" hadoop_add_option "--workers" "turn on worker mode" + hadoop_add_subcommand "apiserver" "run yarn-native-service rest server" hadoop_add_subcommand "application" client "prints application(s) report/kill application" hadoop_add_subcommand "applicationattempt" client "prints applicationattempt(s) report" hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries" @@ -52,6 +53,7 @@ function hadoop_usage hadoop_add_subcommand "apiserver" "run yarn-native-service rest server" hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon" hadoop_add_subcommand "service" "run a service" + hadoop_add_subcommand "sharedcachemanager" admin "run the SharedCacheManager daemon" hadoop_add_subcommand "timelinereader" client "run the timeline reader server" hadoop_add_subcommand "timelineserver" daemon "run the timeline server" hadoop_add_subcommand "top" client "view cluster information" @@ -70,6 +72,18 @@ function yarncmd_case shift case ${subcmd} in + apiserver) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApiServerWebApp' + local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\ +${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\ +${HADOOP_HDFS_HOME}/${HDFS_DIR},\ +${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\ +${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\ +${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" + hadoop_translate_cygwin_path sld + hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}" + ;; application|applicationattempt|container) HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI set -- "${subcmd}" "$@" @@ -151,14 +165,10 @@ function yarncmd_case scmadmin) HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin' ;; - apiserver) - HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services"'/*' - hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*' - HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApiServerWebApp' + service) + HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI' local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\ ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\ -${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services,\ ${HADOOP_HDFS_HOME}/${HDFS_DIR},\ ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\ @@ -170,19 +180,6 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager' ;; - service) - hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services"'/*' - HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI' - local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\ -${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\ -${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services,\ -${HADOOP_HDFS_HOME}/${HDFS_DIR},\ -${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\ -${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\ -${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" - hadoop_translate_cygwin_path sld - hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}" - ;; timelinereader) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index 7d9f15cedf..1077ccda82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -23,7 +23,6 @@ hadoop-yarn-services-api Apache Hadoop YARN Services API - 3.0.0-beta1-SNAPSHOT jar Hadoop YARN REST APIs for services @@ -46,7 +45,6 @@ org.apache.maven.plugins maven-jar-plugin - ${maven-jar-plugin.version} @@ -59,9 +57,6 @@ - - **/run_rest_service.sh - @@ -92,97 +87,34 @@ org.apache.hadoop hadoop-yarn-services-core - ${project.version} - io.swagger - swagger-annotations + org.apache.hadoop + hadoop-yarn-api - com.fasterxml.jackson.core - jackson-core + org.apache.hadoop + hadoop-yarn-common - com.fasterxml.jackson.core - jackson-annotations + org.apache.hadoop + hadoop-common - com.fasterxml.jackson.core - jackson-databind + org.slf4j + slf4j-api - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider + org.eclipse.jetty + jetty-webapp + + + com.google.inject + guice + + + javax.ws.rs + jsr311-api - - - - dist - - false - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - org.apache.hadoop - hadoop-assemblies - ${project.version} - - - - - dist - prepare-package - - single - - - false - false - ${project.artifactId}-${project.version} - - hadoop-yarn-services-api - - - - - - - - - - - rat - - - - org.apache.rat - apache-rat-plugin - ${apache-rat-plugin.version} - - - check-licenses - - check - - - - - - **/*.json - **/THIRD-PARTY.properties - - - - - - - - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java index b226df7917..fc65a63cc1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java @@ -59,8 +59,10 @@ public class ApiServerWebApp extends AbstractService { public static void main(String[] args) throws IOException { ApiServerWebApp apiWebApp = new ApiServerWebApp(); try { - apiWebApp.startWebApp(); + apiWebApp.init(new YarnConfiguration()); + apiWebApp.serviceStart(); } catch (Exception e) { + logger.error("Got exception starting", e); apiWebApp.close(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index 1f8a408a63..d9b7adb753 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -104,7 +104,6 @@ com.beust jcommander - 1.30 @@ -126,37 +125,16 @@ org.codehaus.jackson jackson-core-asl - compile - - - - org.codehaus.jackson - jackson-jaxrs - compile org.codehaus.jackson jackson-mapper-asl - compile - org.codehaus.jackson - jackson-xc - compile - - - - org.apache.hadoop - hadoop-common - test-jar - test - - - - org.apache.hadoop - hadoop-hdfs + com.fasterxml.jackson.core + jackson-annotations @@ -167,25 +145,31 @@ org.apache.hadoop hadoop-yarn-client - compile - - - - org.apache.hadoop - hadoop-yarn-server-web-proxy - compile org.apache.hadoop hadoop-yarn-registry - compile - junit - junit - test + org.apache.hadoop + hadoop-yarn-common + + + + org.apache.hadoop + hadoop-common + + + + org.apache.hadoop + hadoop-annotations + + + + org.apache.hadoop + hadoop-yarn-api @@ -195,13 +179,12 @@ org.apache.commons - commons-compress + commons-configuration2 - commons-digester - commons-digester - 1.8 + org.apache.commons + commons-compress @@ -215,37 +198,13 @@ - commons-logging - commons-logging + org.apache.curator + curator-client - com.codahale.metrics - metrics-core - - - - com.codahale.metrics - metrics-servlets - 3.0.1 - - - - - - - - org.apache.zookeeper - zookeeper - - - - - - - - javax.servlet - javax.servlet-api + org.apache.curator + curator-framework @@ -254,38 +213,23 @@ - com.sun.jersey - jersey-client + org.yaml + snakeyaml - com.sun.jersey - jersey-json + io.swagger + swagger-annotations - - com.sun.jersey - jersey-server - + + + - com.google.inject - guice - - - - com.google.code.gson - gson - - - - com.google.inject.extensions - guice-servlet - - - - com.sun.jersey.contribs - jersey-guice + junit + junit + test @@ -294,115 +238,24 @@ test - - org.easymock - easymock - 3.1 - test - - - org.objenesis - objenesis - - - - - - org.powermock - powermock-api-easymock - 1.6.5 - test - - - - org.powermock - powermock-module-junit4 - 1.6.5 - - - org.javassist - javassist - - - org.objenesis - objenesis - - - - - - javax.servlet.jsp - jsp-api - runtime - - - - org.codehaus.jettison - jettison - - - - org.yaml - snakeyaml - 1.16 - compile - - - - io.swagger - swagger-annotations - 1.5.4 - - org.apache.hadoop hadoop-minicluster test + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + test + + + + org.apache.curator + curator-test + test + + - - - - dist - - false - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - org.apache.hadoop - hadoop-assemblies - ${project.version} - - - - - dist - prepare-package - - single - - - false - false - ${project.artifactId}-${project.version} - - hadoop-yarn-services-dist - - - - - - - - - - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java deleted file mode 100644 index 061121effc..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.client.params; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import org.apache.hadoop.yarn.service.utils.SliderUtils; -import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; -import org.apache.hadoop.yarn.service.exceptions.UsageException; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -@Parameters(commandNames = { SliderActions.ACTION_KDIAG}, - commandDescription = SliderActions.DESCRIBE_ACTION_KDIAG) - -public class ActionKDiagArgs extends AbstractActionArgs { - - @Override - public String getActionName() { - return SliderActions.ACTION_KDIAG; - } - - @Parameter(names = {ARG_SERVICES}, variableArity = true, - description =" list of services to check") - public List services = new ArrayList<>(); - - @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT}, - description = "output file for report") - public File out; - - @Parameter(names = {ARG_KEYTAB}, description = "keytab to use") - public File keytab; - - @Parameter(names = {ARG_KEYLEN}, description = "minimum key length") - public int keylen = 256; - - @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab") - public String principal; - - @Parameter(names = {ARG_SECURE}, description = "Is security required") - public boolean secure = false; - - @Override - public int getMinParams() { - return 0; - } - - @Override - public void validate() throws BadCommandArgumentsException, UsageException { - super.validate(); - if (keytab != null && SliderUtils.isUnset(principal)) { - throw new UsageException("Missing argument " + ARG_PRINCIPAL); - } - if (keytab == null && SliderUtils.isSet(principal)) { - throw new UsageException("Missing argument " + ARG_KEYTAB); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java index 7b957fa504..09cae249f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java @@ -47,7 +47,6 @@ public class ClientArgs extends CommonArgs { private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs(); private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs(); private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs(); - private final ActionKDiagArgs actionKDiagArgs = new ActionKDiagArgs(); private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs(); private final ActionListArgs actionListArgs = new ActionListArgs(); private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs(); @@ -207,10 +206,6 @@ public void applyAction() throws SliderException { bindCoreAction(actionHelpArgs); break; - case ACTION_KDIAG: - bindCoreAction(actionKDiagArgs); - break; - case ACTION_KEYTAB: bindCoreAction(actionKeytabArgs); break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java index add2475c38..62d7a6a35e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java @@ -19,8 +19,6 @@ package org.apache.hadoop.yarn.service.registry; import com.google.common.base.Preconditions; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -32,6 +30,8 @@ import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId; import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; @@ -43,8 +43,8 @@ * is registered, offers access to the record and other things. */ public class YarnRegistryViewForProviders { - private static final Log LOG = - LogFactory.getLog(YarnRegistryViewForProviders.class); + private static final Logger LOG = + LoggerFactory.getLogger(YarnRegistryViewForProviders.class); private final RegistryOperations registryOperations; private final String user; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java deleted file mode 100644 index c0712c3e67..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java +++ /dev/null @@ -1,680 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.SaslPropertiesResolver; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.util.ExitUtil; -import org.apache.hadoop.util.Shell; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.crypto.Cipher; -import java.io.Closeable; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.lang.reflect.InvocationTargetException; -import java.net.InetAddress; -import java.security.NoSuchAlgorithmException; -import java.util.Collection; -import java.util.Date; -import java.util.List; -import java.util.regex.Pattern; - -import static org.apache.hadoop.security.UserGroupInformation.*; -import static org.apache.hadoop.security.authentication.util.KerberosUtil.*; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*; - -/** - * Kerberos diagnostics - * At some point this may move to hadoop core, so please keep use of slider - * methods and classes to ~0. - * - * This operation expands some of the diagnostic output of the security code, - * but not all. For completeness - * - * Set the environment variable {@code HADOOP_JAAS_DEBUG=true} - * Set the log level for {@code org.apache.hadoop.security=DEBUG} - */ -public class KerberosDiags implements Closeable { - - private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class); - public static final String KRB5_CCNAME = "KRB5CCNAME"; - public static final String JAVA_SECURITY_KRB5_CONF - = "java.security.krb5.conf"; - public static final String JAVA_SECURITY_KRB5_REALM - = "java.security.krb5.realm"; - public static final String SUN_SECURITY_KRB5_DEBUG - = "sun.security.krb5.debug"; - public static final String SUN_SECURITY_SPNEGO_DEBUG - = "sun.security.spnego.debug"; - public static final String SUN_SECURITY_JAAS_FILE - = "java.security.auth.login.config"; - public static final String KERBEROS_KINIT_COMMAND - = "hadoop.kerberos.kinit.command"; - public static final String HADOOP_AUTHENTICATION_IS_DISABLED - = "Hadoop authentication is disabled"; - public static final String UNSET = "(unset)"; - public static final String NO_DEFAULT_REALM = "Cannot locate default realm"; - - private final Configuration conf; - private final List services; - private final PrintStream out; - private final File keytab; - private final String principal; - private final long minKeyLength; - private final boolean securityRequired; - - public static final String CAT_JVM = "JVM"; - public static final String CAT_JAAS = "JAAS"; - public static final String CAT_CONFIG = "CONFIG"; - public static final String CAT_LOGIN = "LOGIN"; - public static final String CAT_KERBEROS = "KERBEROS"; - public static final String CAT_SASL = "SASL"; - - @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") - public KerberosDiags(Configuration conf, - PrintStream out, - List services, - File keytab, - String principal, - long minKeyLength, - boolean securityRequired) { - this.conf = conf; - this.services = services; - this.keytab = keytab; - this.principal = principal; - this.out = out; - this.minKeyLength = minKeyLength; - this.securityRequired = securityRequired; - } - - @Override - public void close() throws IOException { - flush(); - } - - /** - * Execute diagnostics. - *

- * Things it would be nice if UGI made accessible - *

    - *
  1. A way to enable JAAS debug programatically
    2. - *
  2. Access to the TGT
    3. - *
- * @return true if security was enabled and all probes were successful - * @throws KerberosDiagsFailure explicitly raised failure - * @throws Exception other security problems - */ - @SuppressWarnings("deprecation") - public boolean execute() throws Exception { - - title("Kerberos Diagnostics scan at %s", - new Date(System.currentTimeMillis())); - - // check that the machine has a name - println("Hostname: %s", - InetAddress.getLocalHost().getCanonicalHostName()); - - // Fail fast on a JVM without JCE installed. - validateKeyLength(); - - // look at realm - println("JVM Kerberos Login Module = %s", getKrb5LoginModuleName()); - printDefaultRealm(); - - title("System Properties"); - for (String prop : new String[]{ - JAVA_SECURITY_KRB5_CONF, - JAVA_SECURITY_KRB5_REALM, - SUN_SECURITY_KRB5_DEBUG, - SUN_SECURITY_SPNEGO_DEBUG, - SUN_SECURITY_JAAS_FILE - }) { - printSysprop(prop); - } - - title("Environment Variables"); - for (String env : new String[]{ - "HADOOP_JAAS_DEBUG", - KRB5_CCNAME, - "HADOOP_USER_NAME", - "HADOOP_PROXY_USER", - HADOOP_TOKEN_FILE_LOCATION, - }) { - printEnv(env); - } - - for (String prop : new String[]{ - KERBEROS_KINIT_COMMAND, - HADOOP_SECURITY_AUTHENTICATION, - HADOOP_SECURITY_AUTHORIZATION, - "hadoop.kerberos.min.seconds.before.relogin", // not in 2.6 - "hadoop.security.dns.interface", // not in 2.6 - "hadoop.security.dns.nameserver", // not in 2.6 - HADOOP_RPC_PROTECTION, - HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS, - HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX, - HADOOP_SECURITY_GROUP_MAPPING, - "hadoop.security.impersonation.provider.class", // not in 2.6 - "dfs.data.transfer.protection" // HDFS - }) { - printConfOpt(prop); - } - - // check that authentication is enabled - if (SecurityUtil.getAuthenticationMethod(conf) - .equals(AuthenticationMethod.SIMPLE)) { - println(HADOOP_AUTHENTICATION_IS_DISABLED); - failif(securityRequired, CAT_CONFIG, HADOOP_AUTHENTICATION_IS_DISABLED); - // no security, skip rest of test - return false; - } - - validateKrb5File(); - validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS); - validateSasl("dfs.data.transfer.saslproperties.resolver.class"); - validateKinitExecutable(); - validateJAAS(); - // now the big test: login, then try again - boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG); - boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG); - try { - title("Logging in"); - - if (keytab != null) { - dumpKeytab(keytab); - loginFromKeytab(); - } else { - UserGroupInformation loginUser = getLoginUser(); - dumpUGI("Log in user", loginUser); - validateUGI("Login user", loginUser); - println("Ticket based login: %b", isLoginTicketBased()); - println("Keytab based login: %b", isLoginKeytabBased()); - } - - return true; - } finally { - // restore original system properties - System.setProperty(SUN_SECURITY_KRB5_DEBUG, - Boolean.toString(krb5Debug)); - System.setProperty(SUN_SECURITY_SPNEGO_DEBUG, - Boolean.toString(spnegoDebug)); - } - } - - /** - * Fail fast on a JVM without JCE installed. - * - * This is a recurrent problem - * (that is: it keeps creeping back with JVM updates); - * a fast failure is the best tactic - * @throws NoSuchAlgorithmException - */ - - protected void validateKeyLength() throws NoSuchAlgorithmException { - int aesLen = Cipher.getMaxAllowedKeyLength("AES"); - println("Maximum AES encryption key length %d bits", aesLen); - failif (aesLen < minKeyLength, - CAT_JVM, - "Java Cryptography Extensions are not installed on this JVM." - +" Maximum supported key length %s - minimum required %d", - aesLen, minKeyLength); - } - - /** - * Get the default realm. - *

- * Not having a default realm may be harmless, so is noted at info. - * All other invocation failures are downgraded to warn, as - * follow-on actions may still work. - * failure to invoke the method via introspection is rejected, - * as it's a sign of JVM compatibility issues that may have other - * consequences - */ - protected void printDefaultRealm() { - try { - println("Default Realm = %s", - getDefaultRealm()); - } catch (ClassNotFoundException - | IllegalAccessException - | NoSuchMethodException e) { - - throw new KerberosDiagsFailure(CAT_JVM, e, - "Failed to invoke krb5.Config.getDefaultRealm: %s", e); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause() != null ? e.getCause() : e; - if (cause.toString().contains(NO_DEFAULT_REALM)) { - // exception raised if there is no default realm. This is not - // always a problem, so downgrade to a message. - println("Host has no default realm"); - LOG.debug(cause.toString(), cause); - } else { - println("Kerberos.getDefaultRealm() failed: %s\n%s", - cause, - org.apache.hadoop.util.StringUtils.stringifyException(cause)); - } - } - } - - /** - * Locate the krb5.conf file and dump it. - * No-op on windows. - * @throws IOException - */ - private void validateKrb5File() throws IOException { - if (!Shell.WINDOWS) { - title("Locating Kerberos configuration file"); - String krbPath = "/etc/krb5.conf"; - String jvmKrbPath = System.getProperty(JAVA_SECURITY_KRB5_CONF); - if (jvmKrbPath != null) { - println("Setting kerberos path from sysprop %s: %s", - JAVA_SECURITY_KRB5_CONF, jvmKrbPath); - krbPath = jvmKrbPath; - } - - String krb5name = System.getenv(KRB5_CCNAME); - if (krb5name != null) { - println("Setting kerberos path from environment variable %s: %s", - KRB5_CCNAME, krb5name); - krbPath = krb5name; - if (jvmKrbPath != null) { - println("Warning - both %s and %s were set - %s takes priority", - JAVA_SECURITY_KRB5_CONF, KRB5_CCNAME, KRB5_CCNAME); - } - } - - File krbFile = new File(krbPath); - println("Kerberos configuration file = %s", krbFile); - failif(!krbFile.exists(), - CAT_KERBEROS, - "Kerberos configuration file %s not found", krbFile); - dump(krbFile); - } - } - - /** - * Dump a keytab: list all principals. - * @param keytabFile the keytab file - * @throws IOException IO problems - */ - public void dumpKeytab(File keytabFile) throws IOException { - title("Examining keytab %s", keytabFile); - File kt = keytabFile.getCanonicalFile(); - failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt); - failif(!kt.isFile(), CAT_CONFIG, "Keytab is not a valid file: %s", kt); - - String[] names = getPrincipalNames(keytabFile.getCanonicalPath(), - Pattern.compile(".*")); - println("keytab entry count: %d", names.length); - for (String name : names) { - println(" %s", name); - } - println("-----"); - } - - /** - * Log in from a keytab, dump the UGI, validate it, then try and log in again. - * That second-time login catches JVM/Hadoop compatibility problems. - * @throws IOException - */ - private void loginFromKeytab() throws IOException { - UserGroupInformation ugi; - String identity; - if (keytab != null) { - File kt = keytab.getCanonicalFile(); - println("Using keytab %s principal %s", kt, principal); - identity = principal; - - failif(StringUtils.isEmpty(principal), CAT_KERBEROS, - "No principal defined"); - ugi = loginUserFromKeytabAndReturnUGI(principal, kt.getPath()); - dumpUGI(identity, ugi); - validateUGI(principal, ugi); - - title("Attempting to log in from keytab again"); - // package scoped -hence the reason why this class must be in the - // hadoop.security package - setShouldRenewImmediatelyForTests(true); - // attempt a new login - ugi.reloginFromKeytab(); - } else { - println("No keytab: logging is as current user"); - } - } - - /** - * Dump a UGI. - * @param title title of this section - * @param ugi UGI to dump - * @throws IOException - */ - private void dumpUGI(String title, UserGroupInformation ugi) - throws IOException { - title(title); - println("UGI instance = %s", ugi); - println("Has kerberos credentials: %b", ugi.hasKerberosCredentials()); - println("Authentication method: %s", ugi.getAuthenticationMethod()); - println("Real Authentication method: %s", - ugi.getRealAuthenticationMethod()); - title("Group names"); - for (String name : ugi.getGroupNames()) { - println(name); - } - title("Credentials"); - Credentials credentials = ugi.getCredentials(); - List secretKeys = credentials.getAllSecretKeys(); - title("Secret keys"); - if (!secretKeys.isEmpty()) { - for (Text secret: secretKeys) { - println("%s", secret); - } - } else { - println("(none)"); - } - - dumpTokens(ugi); - } - - /** - * Validate the UGI: verify it is kerberized. - * @param messagePrefix message in exceptions - * @param user user to validate - */ - private void validateUGI(String messagePrefix, UserGroupInformation user) { - failif(!user.hasKerberosCredentials(), - CAT_LOGIN, "%s: No kerberos credentials for %s", messagePrefix, user); - failif(user.getAuthenticationMethod() == null, - CAT_LOGIN, "%s: Null AuthenticationMethod for %s", messagePrefix, user); - } - - /** - * A cursory look at the {@code kinit} executable. - * If it is an absolute path: it must exist with a size > 0. - * If it is just a command, it has to be on the path. There's no check - * for that -but the PATH is printed out. - */ - private void validateKinitExecutable() { - String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, ""); - if (!kinit.isEmpty()) { - File kinitPath = new File(kinit); - println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath); - if (kinitPath.isAbsolute()) { - failif(!kinitPath.exists(), CAT_KERBEROS, - "%s executable does not exist: %s", - KERBEROS_KINIT_COMMAND, kinitPath); - failif(!kinitPath.isFile(), CAT_KERBEROS, - "%s path does not refer to a file: %s", - KERBEROS_KINIT_COMMAND, kinitPath); - failif(kinitPath.length() == 0, CAT_KERBEROS, - "%s file is empty: %s", - KERBEROS_KINIT_COMMAND, kinitPath); - } else { - println("Executable %s is relative -must be on the PATH", kinit); - printEnv("PATH"); - } - } - } - - /** - * Try to load the SASL resolver. - * @param saslPropsResolverKey key for the SASL resolver - */ - private void validateSasl(String saslPropsResolverKey) { - title("Resolving SASL property %s", saslPropsResolverKey); - String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey); - try { - Class resolverClass = conf.getClass( - saslPropsResolverKey, - SaslPropertiesResolver.class, SaslPropertiesResolver.class); - println("Resolver is %s", resolverClass); - } catch (RuntimeException e) { - throw new KerberosDiagsFailure(CAT_SASL, e, - "Failed to load %s class %s", - saslPropsResolverKey, saslPropsResolver); - } - } - - /** - * Validate any JAAS entry referenced in the {@link #SUN_SECURITY_JAAS_FILE} - * property. - */ - private void validateJAAS() { - String jaasFilename = System.getProperty(SUN_SECURITY_JAAS_FILE); - if (jaasFilename != null) { - title("JAAS"); - File jaasFile = new File(jaasFilename); - println("JAAS file is defined in %s: %s", - SUN_SECURITY_JAAS_FILE, jaasFile); - failif(!jaasFile.exists(), CAT_JAAS, - "JAAS file does not exist: %s", jaasFile); - failif(!jaasFile.isFile(), CAT_JAAS, - "Specified JAAS file is not a file: %s", jaasFile); - } - } - - /** - * Dump all tokens of a user - * @param user user - */ - public void dumpTokens(UserGroupInformation user) { - Collection> tokens - = user.getCredentials().getAllTokens(); - title("Token Count: %d", tokens.size()); - for (Token token : tokens) { - println("Token %s", token.getKind()); - } - } - - /** - * Set the System property to true; return the old value for caching - * @param sysprop property - * @return the previous value - */ - private boolean getAndSet(String sysprop) { - boolean old = Boolean.getBoolean(sysprop); - System.setProperty(sysprop, "true"); - return old; - } - - /** - * Flush all active output channels, including {@Code System.err}, - * so as to stay in sync with any JRE log messages. - */ - private void flush() { - if (out != null) { - out.flush(); - } else { - System.out.flush(); - } - System.err.flush(); - } - - /** - * Format and print a line of output. - * This goes to any output file, or - * is logged at info. The output is flushed before and after, to - * try and stay in sync with JRE logging. - * @param format format string - * @param args any arguments - */ - @VisibleForTesting - public void println(String format, Object... args) { - println(format(format, args)); - } - - /** - * Print a line of output. This goes to any output file, or - * is logged at info. The output is flushed before and after, to - * try and stay in sync with JRE logging. - * @param msg message string - */ - @VisibleForTesting - private void println(String msg) { - flush(); - if (out != null) { - out.println(msg); - } else { - LOG.info(msg); - } - flush(); - } - - /** - * Print a title entry - * @param format format string - * @param args any arguments - */ - private void title(String format, Object... args) { - println(""); - println(""); - String msg = "== " + format(format, args) + " =="; - println(msg); - println(""); - } - - /** - * Print a system property, or {@link #UNSET} if unset. - * @param property property to print - */ - private void printSysprop(String property) { - println("%s = \"%s\"", property, - System.getProperty(property, UNSET)); - } - - /** - * Print a configuration option, or {@link #UNSET} if unset. - * @param option option to print - */ - private void printConfOpt(String option) { - println("%s = \"%s\"", option, conf.get(option, UNSET)); - } - - /** - * Print an environment variable's name and value; printing - * {@link #UNSET} if it is not set - * @param variable environment variable - */ - private void printEnv(String variable) { - String env = System.getenv(variable); - println("%s = \"%s\"", variable, env != null ? env : UNSET); - } - - /** - * Dump any file to standard out; add a trailing newline - * @param file file to dump - * @throws IOException IO problems - */ - public void dump(File file) throws IOException { - try (FileInputStream in = new FileInputStream(file)) { - for (String line : IOUtils.readLines(in)) { - println("%s", line); - } - } - println(""); - } - - /** - * Format and raise a failure - * - * @param category category for exception - * @param message string formatting message - * @param args any arguments for the formatting - * @throws KerberosDiagsFailure containing the formatted text - */ - private void fail(String category, String message, Object... args) - throws KerberosDiagsFailure { - throw new KerberosDiagsFailure(category, message, args); - } - - /** - * Conditional failure with string formatted arguments - * @param condition failure condition - * @param category category for exception - * @param message string formatting message - * @param args any arguments for the formatting - * @throws KerberosDiagsFailure containing the formatted text - * if the condition was met - */ - private void failif(boolean condition, - String category, - String message, - Object... args) - throws KerberosDiagsFailure { - if (condition) { - fail(category, message, args); - } - } - - /** - * Format a string, treating a call where there are no varags values - * as a string to pass through unformatted. - * @param message message, which is either a format string + args, or - * a general string - * @param args argument array - * @return a string for printing. - */ - public static String format(String message, Object... args) { - if (args.length == 0) { - return message; - } else { - return String.format(message, args); - } - } - - /** - * Diagnostics failures return the exit code 41, "unauthorized". - * - * They have a category, initially for testing: the category can be - * validated without having to match on the entire string. - */ - public static class KerberosDiagsFailure extends ExitUtil.ExitException { - private final String category; - - public KerberosDiagsFailure(String category, String message) { - super(41, category + ": " + message); - this.category = category; - } - - public KerberosDiagsFailure(String category, String message, Object... args) { - this(category, format(message, args)); - } - - public KerberosDiagsFailure(String category, Throwable throwable, - String message, Object... args) { - this(category, message, args); - initCause(throwable); - } - - public String getCategory() { - return category; - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java index 6e6f4dd394..7e53d18115 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java @@ -24,35 +24,17 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.GlobFilter; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.io.nativeio.NativeIO; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.ExitUtil; -import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.client.params.Arguments; import org.apache.hadoop.yarn.service.client.params.SliderActions; import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor; import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; -import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; -import org.apache.hadoop.yarn.service.exceptions.BadConfigException; -import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes; import org.apache.hadoop.yarn.service.exceptions.SliderException; -import org.apache.zookeeper.server.util.KerberosUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,32 +45,19 @@ import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; -import java.io.Serializable; -import java.net.InetSocketAddress; import java.net.ServerSocket; -import java.net.Socket; import java.net.URL; import java.net.URLDecoder; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; import java.util.Enumeration; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; import java.util.zip.GZIPOutputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; /** * These are slider-specific Util methods @@ -97,41 +66,6 @@ public final class SliderUtils { private static final Logger log = LoggerFactory.getLogger(SliderUtils.class); - /** - * Atomic bool to track whether or not process security has already been - * turned on (prevents re-entrancy) - */ - private static final AtomicBoolean processSecurityAlreadyInitialized = - new AtomicBoolean(false); - public static final String JAVA_SECURITY_KRB5_REALM = - "java.security.krb5.realm"; - public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc"; - - /** - * Winutils - */ - public static final String WINUTILS = "WINUTILS.EXE"; - /** - * name of openssl program - */ - public static final String OPENSSL = "openssl"; - - /** - * name of python program - */ - public static final String PYTHON = "python"; - - /** - * type of docker standalone service - */ - public static final String DOCKER = "docker"; - /** - * type of docker on yarn service - */ - public static final String DOCKER_YARN = "yarn_docker"; - - public static final int NODE_LIST_LIMIT = 10; - private SliderUtils() { } @@ -152,96 +86,6 @@ public static boolean isEmpty(Collection l) { return l == null || l.isEmpty(); } - /** - * Probe for a collection existing and not being empty - * @param l collection - * @return true if the reference is valid and it contains entries - */ - - public static boolean isNotEmpty(Collection l) { - return l != null && !l.isEmpty(); - } - - /** - * Probe for a map existing and not being empty - * @param m map - * @return true if the reference is valid and it contains map entries - */ - public static boolean isNotEmpty(Map m) { - return m != null && !m.isEmpty(); - } - - /* - * Validates whether num is an integer - * @param num - * @param msg the message to be shown in exception - */ - @SuppressWarnings("ResultOfMethodCallIgnored") - private static void validateNumber(String num, String msg) throws - BadConfigException { - try { - Integer.parseInt(num); - } catch (NumberFormatException nfe) { - throw new BadConfigException(msg + num); - } - } - - /* - * Translates the trailing JVM heapsize unit: g, G, m, M - * This assumes designated unit of 'm' - * @param heapsize - * @return heapsize in MB - */ - public static String translateTrailingHeapUnit(String heapsize) throws - BadConfigException { - String errMsg = "Bad heapsize: "; - if (heapsize.endsWith("m") || heapsize.endsWith("M")) { - String num = heapsize.substring(0, heapsize.length() - 1); - validateNumber(num, errMsg); - return num; - } - if (heapsize.endsWith("g") || heapsize.endsWith("G")) { - String num = heapsize.substring(0, heapsize.length() - 1) + "000"; - validateNumber(num, errMsg); - return num; - } - // check if specified heap size is a number - validateNumber(heapsize, errMsg); - return heapsize; - } - - /** - * recursive directory delete - * @param dir dir to delete - * @throws IOException on any problem - */ - public static void deleteDirectoryTree(File dir) throws IOException { - if (dir.exists()) { - if (dir.isDirectory()) { - log.info("Cleaning up {}", dir); - //delete the children - File[] files = dir.listFiles(); - if (files == null) { - throw new IOException("listfiles() failed for " + dir); - } - for (File file : files) { - log.info("deleting {}", file); - if (!file.delete()) { - log.warn("Unable to delete " + file); - } - } - if (!dir.delete()) { - log.warn("Unable to delete " + dir); - } - } else { - throw new IOException("Not a directory " + dir); - } - } else { - //not found, do nothing - log.debug("No output dir yet"); - } - } - /** * Find a containing JAR * @param clazz class to find @@ -298,127 +142,6 @@ public static File findContainingJar(Class my_class) throws IOException { return null; } - public static void checkPort(String hostname, int port, int connectTimeout) - throws IOException { - InetSocketAddress addr = new InetSocketAddress(hostname, port); - checkPort(hostname, addr, connectTimeout); - } - - @SuppressWarnings("SocketOpenedButNotSafelyClosed") - public static void checkPort(String name, - InetSocketAddress address, - int connectTimeout) - throws IOException { - try(Socket socket = new Socket()) { - socket.connect(address, connectTimeout); - } catch (Exception e) { - throw new IOException("Failed to connect to " + name - + " at " + address - + " after " + connectTimeout + "milliseconds" - + ": " + e, - e); - } - } - - public static void checkURL(String name, String url, int timeout) throws - IOException { - InetSocketAddress address = NetUtils.createSocketAddr(url); - checkPort(name, address, timeout); - } - - /** - * A required file - * @param role role of the file (for errors) - * @param filename the filename - * @throws ExitUtil.ExitException if the file is missing - * @return the file - */ - public static File requiredFile(String filename, String role) throws - IOException { - if (filename.isEmpty()) { - throw new ExitUtil.ExitException(-1, role + " file not defined"); - } - File file = new File(filename); - if (!file.exists()) { - throw new ExitUtil.ExitException(-1, - role + " file not found: " + - file.getCanonicalPath()); - } - return file; - } - - private static final PatternValidator clusternamePattern - = new PatternValidator("[a-z][a-z0-9_-]*"); - - /** - * Normalize a cluster name then verify that it is valid - * @param name proposed cluster name - * @return true iff it is valid - */ - public static boolean isClusternameValid(String name) { - return name != null && clusternamePattern.matches(name); - } - - /** - * Copy a directory to a new FS -both paths must be qualified. If - * a directory needs to be created, supplied permissions can override - * the default values. Existing directories are not touched - * @param conf conf file - * @param srcDirPath src dir - * @param destDirPath dest dir - * @param permission permission for the dest directory; null means "default" - * @return # of files copies - */ - @SuppressWarnings("deprecation") - public static int copyDirectory(Configuration conf, - Path srcDirPath, - Path destDirPath, - FsPermission permission) throws - IOException, - BadClusterStateException { - FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf); - FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf); - //list all paths in the src. - if (!srcFS.exists(srcDirPath)) { - throw new FileNotFoundException("Source dir not found " + srcDirPath); - } - if (!srcFS.isDirectory(srcDirPath)) { - throw new FileNotFoundException( - "Source dir not a directory " + srcDirPath); - } - GlobFilter dotFilter = new GlobFilter("[!.]*"); - FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter); - int srcFileCount = entries.length; - if (srcFileCount == 0) { - return 0; - } - if (permission == null) { - permission = FsPermission.getDirDefault(); - } - if (!destFS.exists(destDirPath)) { - new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath, - permission); - } - Path[] sourcePaths = new Path[srcFileCount]; - for (int i = 0; i < srcFileCount; i++) { - FileStatus e = entries[i]; - Path srcFile = e.getPath(); - if (srcFS.isDirectory(srcFile)) { - String msg = "Configuration dir " + srcDirPath - + " contains a directory " + srcFile; - log.warn(msg); - throw new IOException(msg); - } - log.debug("copying src conf file {}", srcFile); - sourcePaths[i] = srcFile; - } - log.debug("Copying {} files from {} to dest {}", srcFileCount, - srcDirPath, - destDirPath); - FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf); - return srcFileCount; - } - /** * Copy a file to a new FS -both paths must be qualified. * @param conf conf file @@ -495,19 +218,6 @@ public static String join(Collection collection, s : (b.substring(0, b.length() - length)); } - /** - * Join an array of strings with a separator that appears after every - * instance in the list -including at the end - * @param collection strings - * @param separator separator string - * @return the joined entries - */ - public static String join(String[] collection, String separator) { - return join(collection, separator, true); - - - } - /** * Join an array of strings with a separator that appears after every * instance in the list -optionally at the end @@ -521,30 +231,6 @@ public static String join(String[] collection, String separator, return join(Arrays.asList(collection), separator, trailing); } - /** - * Join an array of strings with a separator that appears after every - * instance in the list -except at the end - * @param collection strings - * @param separator separator string - * @return the list - */ - public static String joinWithInnerSeparator(String separator, - Object... collection) { - StringBuilder b = new StringBuilder(); - boolean first = true; - - for (Object o : collection) { - if (first) { - first = false; - } else { - b.append(separator); - } - b.append(o.toString()); - b.append(separator); - } - return b.toString(); - } - /** * Resolve a mandatory environment variable * @param key env var @@ -560,170 +246,6 @@ public static String mandatoryEnvVariable(String key) throws return v; } - public static String appReportToString(ApplicationReport r, - String separator) { - StringBuilder builder = new StringBuilder(512); - builder.append("service ") - .append( - r.getName()) - .append("/") - .append(r.getApplicationType()) - .append(separator); - Set tags = r.getApplicationTags(); - if (!tags.isEmpty()) { - for (String tag : tags) { - builder.append(tag).append(separator); - } - } - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"); - dateFormat.setTimeZone(TimeZone.getDefault()); - builder.append("state: ").append(r.getYarnApplicationState()); - String trackingUrl = r.getTrackingUrl(); - if (isSet(trackingUrl)) { - builder.append(separator).append("URL: ").append(trackingUrl); - } - builder.append(separator) - .append("Started: ") - .append(dateFormat.format(new Date(r.getStartTime()))); - long finishTime = r.getFinishTime(); - if (finishTime > 0) { - builder.append(separator) - .append("Finished: ") - .append(dateFormat.format(new Date(finishTime))); - } - String rpcHost = r.getHost(); - if (!isSet(rpcHost)) { - builder.append(separator) - .append("RPC :") - .append(rpcHost) - .append(':') - .append(r.getRpcPort()); - } - String diagnostics = r.getDiagnostics(); - if (!isSet(diagnostics)) { - builder.append(separator).append("Diagnostics :").append(diagnostics); - } - return builder.toString(); - } - - /** - * Filter a string value given a single filter - * - * @param value - * the string value to check - * @param filter - * a single string filter - * @return return true if value should be trapped, false if it should be let - * through - */ - public static boolean filter(String value, String filter) { - return !(StringUtils.isEmpty(filter) || filter.equals(value)); - } - - /** - * Filter a string value given a set of filters - * - * @param value - * the string value to check - * @param filters - * a set of string filters - * @return return true if value should be trapped, false if it should be let - * through - */ - public static boolean filter(String value, Set filters) { - return !(filters.isEmpty() || filters.contains(value)); - } - - /** - * Sorts the given list of service reports, most recently started - * or finished instance first. - * - * @param instances list of instances - */ - public static void sortApplicationsByMostRecent(List instances) { - Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst()); - } - - /** - * Sorts the given list of service reports - * Finished instances are ordered by finished time and running/accepted instances are - * ordered by start time - * Finally Instance are order by finished instances coming after running instances - * - * @param instances list of instances - */ - public static void sortApplicationReport(List instances) { - if (instances.size() <= 1) { - return; - } - List nonLiveInstance = - new ArrayList<>(instances.size()); - List liveInstance = - new ArrayList<>(instances.size()); - - for (ApplicationReport report : instances) { - if (report.getYarnApplicationState() == YarnApplicationState.RUNNING - || - report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { - liveInstance.add(report); - } else { - nonLiveInstance.add(report); - } - } - - if (liveInstance.size() > 1) { - Collections.sort(liveInstance, new MostRecentlyStartedAppFirst()); - } - if (nonLiveInstance.size() > 1) { - Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst()); - } - instances.clear(); - instances.addAll(liveInstance); - instances.addAll(nonLiveInstance); - } - - /** - * Merge in one map to another -all entries in the second map are - * merged into the first -overwriting any duplicate keys. - * @param first first map -the updated one. - * @param second the map that is merged in - * @return the first map - */ - public static Map mergeMap(Map first, - Map second) { - first.putAll(second); - return first; - } - - /** - * Merge a set of entries into a map. This will take the entryset of - * a map, or a Hadoop collection itself - * @param dest destination - * @param entries entries - * @return dest -with the entries merged in - */ - public static Map mergeEntries(Map dest, - Iterable> entries) { - for (Map.Entry entry : entries) { - dest.put(entry.getKey(), entry.getValue()); - } - return dest; - } - - /** - * Generic map merge logic - * @param first first map - * @param second second map - * @param key type - * @param value type - * @return 'first' merged with the second - */ - public static Map mergeMaps(Map first, - Map second) { - first.putAll(second); - return first; - } - /** * Generic map merge logic * @param first first map @@ -762,125 +284,6 @@ public static String stringifyMap(Map map) { return builder.toString(); } - /** - * Parse an int value, replacing it with defval if undefined; - * @param errorKey key to use in exceptions - * @param defVal default value to use if the key is not in the map - * @param min min value or -1 for do not check - * @param max max value or -1 for do not check - * @return the int value the integer value - * @throws BadConfigException if the value could not be parsed - */ - public static int parseAndValidate(String errorKey, - String valS, - int defVal, - int min, int max) throws - BadConfigException { - if (valS == null) { - valS = Integer.toString(defVal); - } - String trim = valS.trim(); - int val; - try { - val = Integer.decode(trim); - } catch (NumberFormatException e) { - throw new BadConfigException("Failed to parse value of " - + errorKey + ": \"" + trim + "\""); - } - if (min >= 0 && val < min) { - throw new BadConfigException("Value of " - + errorKey + ": " + val + "" - + "is less than the minimum of " + min); - } - if (max >= 0 && val > max) { - throw new BadConfigException("Value of " - + errorKey + ": " + val + "" - + "is more than the maximum of " + max); - } - return val; - } - - public static InetSocketAddress getRmAddress(Configuration conf) { - return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_PORT); - } - - public static InetSocketAddress getRmSchedulerAddress(Configuration conf) { - return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); - } - - /** - * probe to see if the RM scheduler is defined - * @param conf config - * @return true if the RM scheduler address is set to - * something other than 0.0.0.0 - */ - public static boolean isRmSchedulerAddressDefined(Configuration conf) { - InetSocketAddress address = getRmSchedulerAddress(conf); - return isAddressDefined(address); - } - - /** - * probe to see if the address - * @param address network address - * @return true if the scheduler address is set to - * something other than 0.0.0.0 - */ - public static boolean isAddressDefined(InetSocketAddress address) { - if (address == null || address.getHostString() == null) { - return false; - } - return !(address.getHostString().equals("0.0.0.0")); - } - - public static void setRmAddress(Configuration conf, String rmAddr) { - conf.set(YarnConfiguration.RM_ADDRESS, rmAddr); - } - - public static void setRmSchedulerAddress(Configuration conf, String rmAddr) { - conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr); - } - - public static boolean hasAppFinished(ApplicationReport report) { - return report == null || - report.getYarnApplicationState().ordinal() >= - YarnApplicationState.FINISHED.ordinal(); - } - - public static String containerToString(Container container) { - if (container == null) { - return "null container"; - } - return String.format(Locale.ENGLISH, - "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s", - container.getId(), - container.getNodeId(), - container.getNodeHttpAddress(), - container.getPriority(), - container.getResource()); - } - - /** - * convert an AM report to a string for diagnostics - * @param report the report - * @return the string value - */ - public static String reportToString(ApplicationReport report) { - if (report == null) { - return "Null service report"; - } - - return "App " + report.getName() + "/" + report.getApplicationType() + - "# " + - report.getApplicationId() + " user " + report.getUser() + - " is in state " + report.getYarnApplicationState() + - " RPC: " + report.getHost() + ":" + report.getRpcPort() + - " URL: " + report.getOriginalTrackingUrl(); - } - /** * Convert a YARN URL into a string value of a normal URL * @param url URL @@ -949,151 +352,6 @@ public static Map buildEnvMap( return env; } - /** - * Apply a set of command line options to a cluster role map - * @param clusterRoleMap cluster role map to merge onto - * @param commandOptions command opts - */ - public static void applyCommandLineRoleOptsToRoleMap( - Map> clusterRoleMap, - Map> commandOptions) { - for (Map.Entry> entry : commandOptions.entrySet()) { - String key = entry.getKey(); - Map optionMap = entry.getValue(); - Map existingMap = clusterRoleMap.get(key); - if (existingMap == null) { - existingMap = new HashMap(); - } - log.debug("Overwriting role options with command line values {}", - stringifyMap(optionMap)); - mergeMap(existingMap, optionMap); - //set or overwrite the role - clusterRoleMap.put(key, existingMap); - } - } - - /** - * Verify that a Kerberos principal has been set -if not fail - * with an error message that actually tells you what is missing - * @param conf configuration to look at - * @param principal key of principal - * @throws BadConfigException if the key is not set - */ - public static void verifyPrincipalSet(Configuration conf, - String principal) throws - BadConfigException { - String principalName = conf.get(principal); - if (principalName == null) { - throw new BadConfigException("Unset Kerberos principal : %s", - principal); - } - log.debug("Kerberos princial {}={}", principal, principalName); - } - - /** - * Flag to indicate whether the cluster is in secure mode - * @param conf configuration to look at - * @return true if the slider client/service should be in secure mode - */ - public static boolean isHadoopClusterSecure(Configuration conf) { - return SecurityUtil.getAuthenticationMethod(conf) != - UserGroupInformation.AuthenticationMethod.SIMPLE; - } - - /** - * Init security if the cluster configuration declares the cluster is secure - * @param conf configuration to look at - * @return true if the cluster is secure - * @throws IOException cluster is secure - * @throws SliderException the configuration/process is invalid - */ - public static boolean maybeInitSecurity(Configuration conf) throws - IOException, - SliderException { - boolean clusterSecure = isHadoopClusterSecure(conf); - if (clusterSecure) { - log.debug("Enabling security"); - initProcessSecurity(conf); - } - return clusterSecure; - } - - /** - * Turn on security. This is setup to only run once. - * @param conf configuration to build up security - * @return true if security was initialized in this call - * @throws IOException IO/Net problems - * @throws BadConfigException the configuration and system state are inconsistent - */ - public static boolean initProcessSecurity(Configuration conf) throws - IOException, - SliderException { - - if (processSecurityAlreadyInitialized.compareAndSet(true, true)) { - //security is already inited - return false; - } - - log.info("JVM initialized into secure mode with kerberos realm {}", - SliderUtils.getKerberosRealm()); - //this gets UGI to reset its previous world view (i.e simple auth) - //security - log.debug("java.security.krb5.realm={}", - System.getProperty(JAVA_SECURITY_KRB5_REALM, "")); - log.debug("java.security.krb5.kdc={}", - System.getProperty(JAVA_SECURITY_KRB5_KDC, "")); - log.debug("hadoop.security.authentication={}", - conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION)); - log.debug("hadoop.security.authorization={}", - conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION)); - UserGroupInformation.setConfiguration(conf); - UserGroupInformation authUser = UserGroupInformation.getCurrentUser(); - log.debug("Authenticating as {}", authUser); - log.debug("Login user is {}", UserGroupInformation.getLoginUser()); - if (!UserGroupInformation.isSecurityEnabled()) { - throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZED, - "Although secure mode is enabled," + - "the service has already set up its user as an insecure entity %s", - authUser); - } - if (authUser.getAuthenticationMethod() == - UserGroupInformation.AuthenticationMethod.SIMPLE) { - throw new BadConfigException("Auth User is not Kerberized %s" + - " -security has already been set up with the wrong authentication method. " - + "This can occur if a file system has already been created prior to the loading of " - + "the security configuration.", - authUser); - - } - - SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL); - SliderUtils.verifyPrincipalSet(conf, "dfs.namenode.kerberos.principal"); - return true; - } - - /** - * Force an early login: This catches any auth problems early rather than - * in RPC operations - * @throws IOException if the login fails - */ - public static void forceLogin() throws IOException { - if (UserGroupInformation.isSecurityEnabled()) { - if (UserGroupInformation.isLoginKeytabBased()) { - UserGroupInformation.getLoginUser().reloginFromKeytab(); - } else { - UserGroupInformation.getLoginUser().reloginFromTicketCache(); - } - } - } - - public static String getLibDir() { - String[] libDirs = getLibDirs(); - if (libDirs == null || libDirs.length == 0) { - return null; - } - return libDirs[0]; - } - public static String[] getLibDirs() { String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR); if (isUnset(libDirStr)) { @@ -1183,39 +441,6 @@ public static void putAmTarGzipAndUpdate( sliderFileSystem.submitTarGzipAndUpdate(providerResources); } - public static Map> deepClone(Map> src) { - Map> dest = new HashMap<>(); - for (Map.Entry> entry : src.entrySet()) { - dest.put(entry.getKey(), stringMapClone(entry.getValue())); - } - return dest; - } - - public static Map stringMapClone(Map src) { - Map dest = new HashMap<>(); - return mergeEntries(dest, src.entrySet()); - } - - /** - * List a directory in the local filesystem - * @param dir directory - * @return a listing, one to a line - */ - public static String listDir(File dir) { - if (dir == null) { - return ""; - } - String[] confDirEntries = dir.list(); - if (confDirEntries == null) { - return ""; - } - StringBuilder builder = new StringBuilder(); - for (String entry : confDirEntries) { - builder.append(entry).append("\n"); - } - return builder.toString(); - } - /** * Create a file:// path from a local file * @param file file to point the path @@ -1225,16 +450,6 @@ public static Path createLocalPath(File file) { return new Path(file.toURI()); } - public static String getKerberosRealm() { - try { - return KerberosUtil.getDefaultRealm(); - } catch (Exception e) { - log.debug("introspection into JVM internals failed", e); - return "(unknown)"; - - } - } - /** * Build up the classpath for execution * -behaves very differently on a mini test cluster vs a production @@ -1278,171 +493,6 @@ public static ClasspathConstructor buildClasspath(String sliderConfDir, return classpath; } - /** - * Verify that a path refers to a directory. If not - * logs the parent dir then throws an exception - * @param dir the directory - * @param errorlog log for output on an error - * @throws FileNotFoundException if it is not a directory - */ - public static void verifyIsDir(File dir, Logger errorlog) throws - FileNotFoundException { - if (!dir.exists()) { - errorlog.warn("contents of {}: {}", dir, - listDir(dir.getParentFile())); - throw new FileNotFoundException(dir.toString()); - } - if (!dir.isDirectory()) { - errorlog.info("contents of {}: {}", dir, - listDir(dir.getParentFile())); - throw new FileNotFoundException( - "Not a directory: " + dir); - } - } - - /** - * Verify that a file exists - * @param file file - * @param errorlog log for output on an error - * @throws FileNotFoundException - */ - public static void verifyFileExists(File file, Logger errorlog) throws - FileNotFoundException { - if (!file.exists()) { - errorlog.warn("contents of {}: {}", file, - listDir(file.getParentFile())); - throw new FileNotFoundException(file.toString()); - } - if (!file.isFile()) { - throw new FileNotFoundException("Not a file: " + file.toString()); - } - } - - /** - * verify that a config option is set - * @param configuration config - * @param key key - * @return the value, in case it needs to be verified too - * @throws BadConfigException if the key is missing - */ - public static String verifyOptionSet(Configuration configuration, String key, - boolean allowEmpty) throws BadConfigException { - String val = configuration.get(key); - if (val == null) { - throw new BadConfigException( - "Required configuration option \"%s\" not defined ", key); - } - if (!allowEmpty && val.isEmpty()) { - throw new BadConfigException( - "Configuration option \"%s\" must not be empty", key); - } - return val; - } - - /** - * Verify that a keytab property is defined and refers to a non-empty file - * - * @param siteConf configuration - * @param prop property to look for - * @return the file referenced - * @throws BadConfigException on a failure - */ - public static File verifyKeytabExists(Configuration siteConf, - String prop) throws - BadConfigException { - String keytab = siteConf.get(prop); - if (keytab == null) { - throw new BadConfigException("Missing keytab property %s", - prop); - - } - File keytabFile = new File(keytab); - if (!keytabFile.exists()) { - throw new BadConfigException("Missing keytab file %s defined in %s", - keytabFile, - prop); - } - if (keytabFile.length() == 0 || !keytabFile.isFile()) { - throw new BadConfigException("Invalid keytab file %s defined in %s", - keytabFile, - prop); - } - return keytabFile; - } - - /** - * Add a subpath to an existing URL. This extends - * the path, inserting a / between all entries - * if needed. - * @param base base path/URL - * @param path subpath - * @return base+"/"+subpath - */ - public static String appendToURL(String base, String path) { - StringBuilder fullpath = new StringBuilder(base); - if (!base.endsWith("/")) { - fullpath.append("/"); - } - if (path.startsWith("/")) { - fullpath.append(path.substring(1)); - } else { - fullpath.append(path); - } - return fullpath.toString(); - } - - /** - * Truncate the given string to a maximum length provided - * with a pad (...) added to the end if expected size if more than 10. - * @param toTruncate string to truncate; may be null - * @param maxSize maximum size - * @return the truncated/padded string. - */ - public static String truncate(String toTruncate, int maxSize) { - if (toTruncate == null || maxSize < 1 - || toTruncate.length() <= maxSize) { - return toTruncate; - } - - String pad = "..."; - if (maxSize < 10) { - pad = ""; - } - return toTruncate.substring(0, maxSize - pad.length()).concat(pad); - } - - /** - * Given a source folder create zipped file - * - * @param srcFolder - * @param zipFile - * - * @throws IOException - */ - public static void zipFolder(File srcFolder, File zipFile) throws IOException { - log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath()); - List files = new ArrayList<>(); - generateFileList(files, srcFolder, srcFolder, true); - - byte[] buffer = new byte[1024]; - - try (FileOutputStream fos = new FileOutputStream(zipFile)) { - try (ZipOutputStream zos = new ZipOutputStream(fos)) { - - for (String file : files) { - ZipEntry ze = new ZipEntry(file); - zos.putNextEntry(ze); - try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) { - int len; - while ((len = in.read(buffer)) > 0) { - zos.write(buffer, 0, len); - } - } - } - } - } - } - /** * Given a source folder create a tar.gz file * @@ -1478,11 +528,6 @@ public static void tarGzipFolder(String[] libDirs, File tarGzipFile, } } - private static void generateFileList(List fileList, File node, - File rootFolder, Boolean relative) { - generateFileList(fileList, node, rootFolder, relative, null); - } - private static void generateFileList(List fileList, File node, File rootFolder, Boolean relative, FilenameFilter filter) { if (node.isFile()) { @@ -1507,134 +552,6 @@ private static void generateFileList(List fileList, File node, } } - /** - * Check for any needed libraries being present. On Unix none are needed; - * on windows they must be present - * @return true if all is well - */ - public static String checkForRequiredNativeLibraries() { - - if (!Shell.WINDOWS) { - return ""; - } - StringBuilder errorText = new StringBuilder(""); - if (!NativeIO.isAvailable()) { - errorText.append("No native IO library. "); - } - try { - String path = Shell.getQualifiedBinPath(WINUTILS); - log.debug("winutils is at {}", path); - } catch (IOException e) { - errorText.append("No " + WINUTILS); - log.warn("No winutils: {}", e, e); - } - try { - File target = new File("target"); - FileUtil.canRead(target); - } catch (UnsatisfiedLinkError e) { - log.warn("Failing to link to native IO methods: {}", e, e); - errorText.append("No native IO methods"); - } - return errorText.toString(); - } - - /** - * Strictly verify that windows utils is present. - * Checks go as far as opening the file and looking for - * the headers. - * @throws IOException on any problem reading the file - * @throws FileNotFoundException if the file is not considered valid - */ - public static void maybeVerifyWinUtilsValid() throws - IOException, - SliderException { - String errorText = SliderUtils.checkForRequiredNativeLibraries(); - if (!errorText.isEmpty()) { - throw new BadClusterStateException(errorText); - } - } - - /** - * Write bytes to a file - * @param outfile output file - * @param data data to write - * @throws IOException on any IO problem - */ - public static void write(File outfile, byte[] data) - throws IOException { - File parentDir = outfile.getCanonicalFile().getParentFile(); - if (parentDir == null) { - throw new IOException(outfile.getPath() + " has no parent dir"); - } - if (!parentDir.exists()) { - if(!parentDir.mkdirs()) { - throw new IOException("Failed to create parent directory " + parentDir); - } - } - SliderUtils.verifyIsDir(parentDir, log); - try(FileOutputStream out = new FileOutputStream(outfile)) { - out.write(data); - } - } - - /** - * Compare the times of two applications: most recent app comes first - * Specifically: the one whose start time value is greater. - */ - private static class MostRecentlyStartedAppFirst - implements Comparator, Serializable { - @Override - public int compare(ApplicationReport r1, ApplicationReport r2) { - long x = r1.getStartTime(); - long y = r2.getStartTime(); - return compareTwoLongsReverse(x, y); - } - } - - /** - * Compare the times of two applications: most recent app comes first. - * "Recent"== the app whose start time or finish time is the greatest. - */ - private static class MostRecentlyStartedOrFinishedFirst - implements Comparator, Serializable { - @Override - public int compare(ApplicationReport r1, ApplicationReport r2) { - long started1 = r1.getStartTime(); - long started2 = r2.getStartTime(); - long finished1 = r1.getFinishTime(); - long finished2 = r2.getFinishTime(); - long lastEvent1 = Math.max(started1, finished1); - long lastEvent2 = Math.max(started2, finished2); - return compareTwoLongsReverse(lastEvent1, lastEvent2); - } - } - - /** - * Compare the times of two applications: most recently finished app comes first - * Specifically: the one whose finish time value is greater. - */ - private static class MostRecentAppFinishFirst - implements Comparator, Serializable { - @Override - public int compare(ApplicationReport r1, ApplicationReport r2) { - long x = r1.getFinishTime(); - long y = r2.getFinishTime(); - return compareTwoLongsReverse(x, y); - } - } - - /** - * Compare two long values for sorting. As the return value for - * comparators must be int, the simple value of x-y - * is inapplicable - * @param x x value - * @param y y value - * @return +ve if x is less than y, -ve if y is greater than x; 0 for equality - */ - public static int compareTwoLongsReverse(long x, long y) { - return (x < y) ? 1 : ((x == y) ? 0 : -1); - } - public static String createNameTag(String name) { return "Name: " + name; } @@ -1646,9 +563,4 @@ public static String createVersionTag(String version) { public static String createDescriptionTag(String description) { return "Description: " + description; } - - public static final String DAYS = ".days"; - public static final String HOURS = ".hours"; - public static final String MINUTES = ".minutes"; - public static final String SECONDS = ".seconds"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java index 0f4f598562..a2edbc8950 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java @@ -18,12 +18,22 @@ package org.apache.hadoop.yarn.service; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Resource; import org.apache.hadoop.yarn.service.utils.JsonSerDeser; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; import org.codehaus.jackson.map.PropertyNamingStrategy; +import java.io.IOException; + +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class ServiceTestUtils { public static final JsonSerDeser JSON_SER_DESER = @@ -56,4 +66,22 @@ protected Component createComponent(String name, long numContainers, resource.setCpus(1); return comp1; } + + public static SliderFileSystem initMockFs() throws IOException { + return initMockFs(null); + } + + public static SliderFileSystem initMockFs(Service ext) throws IOException { + SliderFileSystem sfs = mock(SliderFileSystem.class); + FileSystem mockFs = mock(FileSystem.class); + JsonSerDeser jsonSerDeser = mock(JsonSerDeser.class); + when(sfs.getFileSystem()).thenReturn(mockFs); + when(sfs.buildClusterDirPath(anyObject())).thenReturn( + new Path("cluster_dir_path")); + if (ext != null) { + when(jsonSerDeser.load(anyObject(), anyObject())).thenReturn(ext); + } + ServiceApiUtil.setJsonSerDeser(jsonSerDeser); + return sfs; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java index be36335a66..959e4d61b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java @@ -18,8 +18,6 @@ package org.apache.hadoop.yarn.service; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.api.records.Service; @@ -27,7 +25,6 @@ import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Resource; -import org.apache.hadoop.yarn.service.utils.JsonSerDeser; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; import org.junit.Assert; @@ -44,10 +41,8 @@ import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME; import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME; import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*; -import static org.easymock.EasyMock.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; /** * Test for ServiceApiUtil helper methods. @@ -78,7 +73,7 @@ public void testResourceValidation() throws Exception { assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR .length()); - SliderFileSystem sfs = initMock(null); + SliderFileSystem sfs = ServiceTestUtils.initMockFs(); Service app = new Service(); @@ -230,7 +225,7 @@ public void testResourceValidation() throws Exception { @Test public void testArtifacts() throws IOException { - SliderFileSystem sfs = initMock(null); + SliderFileSystem sfs = ServiceTestUtils.initMockFs(); Service app = new Service(); app.setName("name"); @@ -309,27 +304,10 @@ private static Service createValidApplication(String compName) { return app; } - private static SliderFileSystem initMock(Service ext) throws IOException { - SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); - FileSystem mockFs = createNiceMock(FileSystem.class); - JsonSerDeser jsonSerDeser = createNiceMock(JsonSerDeser - .class); - expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); - expect(sfs.buildClusterDirPath(anyObject())).andReturn( - new Path("cluster_dir_path")).anyTimes(); - if (ext != null) { - expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext) - .anyTimes(); - } - replay(sfs, mockFs, jsonSerDeser); - ServiceApiUtil.setJsonSerDeser(jsonSerDeser); - return sfs; - } - @Test public void testExternalApplication() throws IOException { Service ext = createValidApplication("comp1"); - SliderFileSystem sfs = initMock(ext); + SliderFileSystem sfs = ServiceTestUtils.initMockFs(ext); Service app = createValidApplication(null); @@ -350,7 +328,7 @@ public void testExternalApplication() throws IOException { @Test public void testDuplicateComponents() throws IOException { - SliderFileSystem sfs = initMock(null); + SliderFileSystem sfs = ServiceTestUtils.initMockFs(); String compName = "comp1"; Service app = createValidApplication(compName); @@ -368,7 +346,7 @@ public void testDuplicateComponents() throws IOException { @Test public void testExternalDuplicateComponent() throws IOException { Service ext = createValidApplication("comp1"); - SliderFileSystem sfs = initMock(ext); + SliderFileSystem sfs = ServiceTestUtils.initMockFs(ext); Service app = createValidApplication("comp1"); Artifact artifact = new Artifact(); @@ -387,7 +365,7 @@ public void testExternalDuplicateComponent() throws IOException { @Test public void testExternalComponent() throws IOException { Service ext = createValidApplication("comp1"); - SliderFileSystem sfs = initMock(ext); + SliderFileSystem sfs = ServiceTestUtils.initMockFs(ext); Service app = createValidApplication("comp2"); Artifact artifact = new Artifact(); @@ -454,7 +432,7 @@ public void testDependencySorting() throws IOException { e)), ex.getMessage()); } - SliderFileSystem sfs = initMock(null); + SliderFileSystem sfs = ServiceTestUtils.initMockFs(); Service service = createValidApplication(null); service.setComponents(Arrays.asList(c, d, e)); try { @@ -470,7 +448,7 @@ public void testDependencySorting() throws IOException { @Test public void testInvalidComponent() throws IOException { - SliderFileSystem sfs = initMock(null); + SliderFileSystem sfs = ServiceTestUtils.initMockFs(); testComponent(sfs); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java index 63aa9c6b48..30f2aeb7a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java @@ -19,8 +19,6 @@ package org.apache.hadoop.yarn.service; import org.apache.commons.io.FileUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.curator.test.TestingCluster; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -51,6 +49,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; import java.io.File; @@ -78,8 +78,8 @@ */ public class TestYarnNativeServices extends ServiceTestUtils{ - private static final Log LOG = - LogFactory.getLog(TestYarnNativeServices.class); + private static final Logger LOG = + LoggerFactory.getLogger(TestYarnNativeServices.class); private MiniYARNCluster yarnCluster = null; private MiniDFSCluster hdfsCluster = null; @@ -416,7 +416,7 @@ private void waitForAllCompToBeReady(ServiceClient client, LOG.info("Num Components " + retrievedApp.getComponents().size()); for (Component component : retrievedApp.getComponents()) { LOG.info("looking for " + component.getName()); - LOG.info(component); + LOG.info(component.toString()); if (component.getContainers() != null) { if (component.getContainers().size() == exampleApp .getComponent(component.getName()).getNumberOfContainers()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java index ecc529dbf8..c53ee2bb83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java @@ -45,7 +45,6 @@ import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS; import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; -import static org.mockito.Matchers.any; import static org.mockito.Mockito.*; public class TestServiceCLI { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java index 04ec52687b..8739382603 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java @@ -18,13 +18,11 @@ package org.apache.hadoop.yarn.service.conf; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.ServiceTestUtils; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.api.records.Configuration; -import org.apache.hadoop.yarn.service.utils.JsonSerDeser; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; import org.junit.Assert; @@ -40,7 +38,6 @@ import java.util.Set; import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.*; -import static org.easymock.EasyMock.*; /** * Test global configuration resolution. @@ -78,12 +75,7 @@ public void testOverride() throws Throwable { assertEquals("1000", worker.getProperty("timeout")); // here is the resolution - SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); - FileSystem mockFs = createNiceMock(FileSystem.class); - expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); - expect(sfs.buildClusterDirPath(anyObject())).andReturn( - new Path("cluster_dir_path")).anyTimes(); - replay(sfs, mockFs); + SliderFileSystem sfs = ServiceTestUtils.initMockFs(); ServiceApiUtil.validateAndResolveService(orig, sfs, new YarnConfiguration()); @@ -162,27 +154,13 @@ public void testOverrideExternalConfiguration() throws IOException { assertEquals(0, other.getProperties().size()); // load the external service - SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); - FileSystem mockFs = createNiceMock(FileSystem.class); - expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); - expect(sfs.buildClusterDirPath(anyObject())).andReturn( - new Path("cluster_dir_path")).anyTimes(); - replay(sfs, mockFs); + SliderFileSystem sfs = ServiceTestUtils.initMockFs(); Service ext = ExampleAppJson.loadResource(APP_JSON); ServiceApiUtil.validateAndResolveService(ext, sfs, new YarnConfiguration()); - reset(sfs, mockFs); // perform the resolution on original service - JsonSerDeser jsonSerDeser = createNiceMock(JsonSerDeser - .class); - expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); - expect(sfs.buildClusterDirPath(anyObject())).andReturn( - new Path("cluster_dir_path")).anyTimes(); - expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext) - .anyTimes(); - replay(sfs, mockFs, jsonSerDeser); - ServiceApiUtil.setJsonSerDeser(jsonSerDeser); + sfs = ServiceTestUtils.initMockFs(ext); ServiceApiUtil.validateAndResolveService(orig, sfs, new YarnConfiguration()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java index 83e9502fd8..a813da3bff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java @@ -18,9 +18,8 @@ package org.apache.hadoop.yarn.service.conf; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.ServiceTestUtils; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; @@ -33,7 +32,6 @@ import java.util.Collection; import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER; -import static org.easymock.EasyMock.*; /** * Test loading example resources. @@ -62,12 +60,7 @@ public void testLoadResource() throws Throwable { try { Service service = JSON_SER_DESER.fromResource(resource); - SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); - FileSystem mockFs = createNiceMock(FileSystem.class); - expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); - expect(sfs.buildClusterDirPath(anyObject())).andReturn( - new Path("cluster_dir_path")).anyTimes(); - replay(sfs, mockFs); + SliderFileSystem sfs = ServiceTestUtils.initMockFs(); ServiceApiUtil.validateAndResolveService(service, sfs, new YarnConfiguration()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java index 5b24a1db10..79406e99e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.service.providers; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.provider.AbstractClientProvider; @@ -29,7 +28,9 @@ import java.util.ArrayList; import java.util.List; -import static org.easymock.EasyMock.*; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Test the AbstractClientProvider shared methods. @@ -55,9 +56,8 @@ protected void validateConfigFile(ConfigFile configFile, @Test public void testConfigFiles() throws IOException { ClientProvider clientProvider = new ClientProvider(); - FileSystem mockFs = createNiceMock(FileSystem.class); - expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes(); - replay(mockFs); + FileSystem mockFs = mock(FileSystem.class); + when(mockFs.exists(anyObject())).thenReturn(true); ConfigFile configFile = new ConfigFile(); List configFiles = new ArrayList<>(); diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 3cbbaa7688..1b3c5f0562 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -82,6 +82,10 @@ org.apache.hadoop hadoop-yarn-server-router + + org.apache.hadoop + hadoop-yarn-services-core +