From 10a4289ebfe8a079562f2006a0e28acdbd8aa0c5 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Wed, 25 Sep 2013 00:35:03 +0000 Subject: [PATCH] YARN-1229. Define constraints on Auxiliary Service names. Change ShuffleHandler service name from mapreduce.shuffle to mapreduce_shuffle. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1526065 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 12 +++++++ .../src/site/apt/ClusterSetup.apt.vm | 2 +- .../src/site/apt/SingleCluster.apt.vm | 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 +++++++ hadoop-mapreduce-project/CHANGES.txt | 12 +++++++ hadoop-mapreduce-project/INSTALL | 4 +-- .../PluggableShuffleAndPluggableSort.apt.vm | 8 ++--- .../apache/hadoop/mapred/ShuffleHandler.java | 2 +- hadoop-yarn-project/CHANGES.txt | 16 ++++++++++ .../src/main/resources/yarn-default.xml | 5 +-- .../containermanager/AuxServices.java | 18 +++++++++++ .../containermanager/TestAuxServices.java | 31 +++++++++++++++++++ 12 files changed, 113 insertions(+), 11 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ad2e6fa969..3da6d1ebec 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -384,6 +384,18 @@ Release 2.2.0 - UNRELEASED BUG FIXES +Release 2.1.2 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 2.1.1-beta - 2013-09-23 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm index cae366abc7..1e2a2c0b99 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm @@ -311,7 +311,7 @@ Hadoop MapReduce Next Generation - Cluster Setup | | | Only applicable if log-aggregation is enabled. | *-------------------------+-------------------------+------------------------+ | <<>> | | | -| | mapreduce.shuffle | | +| | mapreduce_shuffle | | | | | Shuffle service that needs to be set for Map Reduce applications. | *-------------------------+-------------------------+------------------------+ diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm index d7058d9491..c86cd6defd 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm @@ -140,7 +140,7 @@ Add the following configs to your <<>> yarn.nodemanager.aux-services - mapreduce.shuffle + mapreduce_shuffle shuffle service that needs to be set for Map Reduce to run +---+ diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 096ead7f99..0f9d874c83 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -323,6 +323,18 @@ Release 2.2.0 - UNRELEASED OPTIMIZATIONS + BUG FIXES + +Release 2.1.2 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + BUG FIXES HDFS-5139. Remove redundant -R option from setrep. diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 91458f6d63..82a0b8f886 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -191,6 +191,18 @@ Release 2.2.0 - UNRELEASED OPTIMIZATIONS + BUG FIXES + +Release 2.1.2 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + BUG FIXES MAPREDUCE-5504. mapred queue -info inconsistent with types (Kousuke Saruta diff --git a/hadoop-mapreduce-project/INSTALL b/hadoop-mapreduce-project/INSTALL index fae237045b..f52fe2056f 100644 --- a/hadoop-mapreduce-project/INSTALL +++ b/hadoop-mapreduce-project/INSTALL @@ -39,11 +39,11 @@ export YARN_CONF_DIR=$HADOOP_CONF_DIR Step 7) Setup config: for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager. yarn.nodemanager.aux-services - mapreduce.shuffle + mapreduce_shuffle - yarn.nodemanager.aux-services.mapreduce.shuffle.class + yarn.nodemanager.aux-services.mapreduce_shuffle.class org.apache.hadoop.mapred.ShuffleHandler diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm index 8dd2f2ecef..c37fc071d8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm @@ -83,14 +83,14 @@ Hadoop MapReduce Next Generation - Pluggable Shuffle and Pluggable Sort *--------------------------------------+---------------------+-----------------+ | <> | <> | <> | *--------------------------------------+---------------------+-----------------+ -| <<>> | <<<...,mapreduce.shuffle>>> | The auxiliary service name | +| <<>> | <<<...,mapreduce_shuffle>>> | The auxiliary service name | *--------------------------------------+---------------------+-----------------+ -| <<>> | <<>> | The auxiliary service class to use | +| <<>> | <<>> | The auxiliary service class to use | *--------------------------------------+---------------------+-----------------+ <> If setting an auxiliary service in addition the default - <<>> service, then a new service key should be added to the + <<>> service, then a new service key should be added to the <<>> property, for example <<>>. Then the property defining the corresponding class must be - <<>>. + <<>>. \ No newline at end of file diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index f9d1203ba4..82fd59e551 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -146,7 +146,7 @@ public class ShuffleHandler extends AuxiliaryService { private ReadaheadPool readaheadPool = ReadaheadPool.getInstance(); public static final String MAPREDUCE_SHUFFLE_SERVICEID = - "mapreduce.shuffle"; + "mapreduce_shuffle"; private static final Map userRsrc = new ConcurrentHashMap(); diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 204438ba72..37eec561d8 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -56,6 +56,22 @@ Release 2.2.0 - UNRELEASED OPTIMIZATIONS + BUG FIXES + +Release 2.1.2 - UNRELEASED + + INCOMPATIBLE CHANGES + + YARN-1229. Define constraints on Auxiliary Service names. Change + ShuffleHandler service name from mapreduce.shuffle to + mapreduce_shuffle (Xuan Gong via sseth) + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + BUG FIXES YARN-1128. FifoPolicy.computeShares throws NPE on empty list of Schedulables diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 8aa7d01735..65718a9d69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -705,9 +705,10 @@ + the valid service name should only contain a-zA-Z0-9_ and can not start with numbers yarn.nodemanager.aux-services - + @@ -763,7 +764,7 @@ - yarn.nodemanager.aux-services.mapreduce.shuffle.class + yarn.nodemanager.aux-services.mapreduce_shuffle.class org.apache.hadoop.mapred.ShuffleHandler diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index 13f43650a4..0e0e7668f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -40,6 +41,8 @@ import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; import org.apache.hadoop.yarn.server.api.ContainerTerminationContext; +import com.google.common.base.Preconditions; + public class AuxServices extends AbstractService implements ServiceStateChangeListener, EventHandler { @@ -48,6 +51,8 @@ public class AuxServices extends AbstractService protected final Map serviceMap; protected final Map serviceMetaData; + private final Pattern p = Pattern.compile("^[A-Za-z_]+[A-Za-z0-9_]*$"); + public AuxServices() { super(AuxServices.class.getName()); serviceMap = @@ -90,6 +95,13 @@ public void serviceInit(Configuration conf) throws Exception { YarnConfiguration.NM_AUX_SERVICES); for (final String sName : auxNames) { try { + Preconditions + .checkArgument( + validateAuxServiceName(sName), + "The ServiceName: " + sName + " set in " + + YarnConfiguration.NM_AUX_SERVICES +" is invalid." + + "The valid service name should only contain a-zA-Z0-9_ " + + "and can not start with numbers"); Class sClass = conf.getClass( String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, sName), null, AuxiliaryService.class); @@ -199,4 +211,10 @@ public void handle(AuxServicesEvent event) { } } + private boolean validateAuxServiceName(String name) { + if (name == null || name.trim().isEmpty()) { + return false; + } + return p.matcher(name).matches(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java index 81f758ee06..a58f3bda66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java @@ -31,6 +31,8 @@ import java.util.Collection; import java.util.Map; +import junit.framework.Assert; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -288,4 +290,33 @@ public void testAuxUnexpectedStop() { assertTrue(aux.getServices().isEmpty()); } + @Test + public void testValidAuxServiceName() { + final AuxServices aux = new AuxServices(); + Configuration conf = new Configuration(); + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] {"Asrv1", "Bsrv_2"}); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv1"), + ServiceA.class, Service.class); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv_2"), + ServiceB.class, Service.class); + try { + aux.init(conf); + } catch (Exception ex) { + Assert.fail("Should not receive the exception."); + } + + //Test bad auxService Name + final AuxServices aux1 = new AuxServices(); + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] {"1Asrv1"}); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "1Asrv1"), + ServiceA.class, Service.class); + try { + aux1.init(conf); + Assert.fail("Should receive the exception."); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("The ServiceName: 1Asrv1 set in " + + "yarn.nodemanager.aux-services is invalid.The valid service name " + + "should only contain a-zA-Z0-9_ and can not start with numbers")); + } + } }