From cf4631d7c9448205f4952341c071e1650e7922dc Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Mon, 29 Sep 2014 14:36:29 +0000 Subject: [PATCH] HADOOP-11049. javax package system class default is too broad. Contributed by Sangjin Lee --- .../hadoop-common/CHANGES.txt | 3 + .../hadoop/util/ApplicationClassLoader.java | 47 ++++++++++----- ....hadoop.application-classloader.properties | 57 +++++++++++++++++++ .../org/apache/hadoop/util/TestRunJar.java | 2 +- .../hadoop/mapreduce/v2/util/TestMRApps.java | 4 +- .../hadoop/mapreduce/v2/TestMRJobs.java | 2 +- 6 files changed, 98 insertions(+), 17 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/resources/org.apache.hadoop.application-classloader.properties diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 14c21fa4e1..d6be9f024d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -885,6 +885,9 @@ Release 2.6.0 - UNRELEASED HADOOP-11143 NetUtils.wrapException loses inner stack trace on BindException (stevel) + HADOOP-11049. javax package system class default is too broad (Sangjin Lee + via jlowe) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java index 5dda10fc88..b18997c9d9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java @@ -20,12 +20,15 @@ import java.io.File; import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStream; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -45,18 +48,12 @@ public class ApplicationClassLoader extends URLClassLoader { * classes are considered system classes, and are not loaded by the * application classloader. */ - public static final String DEFAULT_SYSTEM_CLASSES = - "java.," + - "javax.," + - "org.w3c.dom.," + - "org.xml.sax.," + - "org.apache.commons.logging.," + - "org.apache.log4j.," + - "org.apache.hadoop.," + - "core-default.xml," + - "hdfs-default.xml," + - "mapred-default.xml," + - "yarn-default.xml"; + public static final String SYSTEM_CLASSES_DEFAULT; + + private static final String PROPERTIES_FILE = + "org.apache.hadoop.application-classloader.properties"; + private static final String SYSTEM_CLASSES_DEFAULT_KEY = + "system.classes.default"; private static final Log LOG = LogFactory.getLog(ApplicationClassLoader.class.getName()); @@ -69,6 +66,30 @@ public boolean accept(File dir, String name) { } }; + static { + InputStream is = null; + try { + is = ApplicationClassLoader.class.getClassLoader(). + getResourceAsStream(PROPERTIES_FILE); + if (is == null) { + throw new ExceptionInInitializerError("properties file " + + PROPERTIES_FILE + " is not found"); + } + Properties props = new Properties(); + props.load(is); + // get the system classes default + String systemClassesDefault = + props.getProperty(SYSTEM_CLASSES_DEFAULT_KEY); + if (systemClassesDefault == null) { + throw new ExceptionInInitializerError("property " + + SYSTEM_CLASSES_DEFAULT_KEY + " is not found"); + } + SYSTEM_CLASSES_DEFAULT = systemClassesDefault; + } catch (IOException e) { + throw new ExceptionInInitializerError(e); + } + } + private final ClassLoader parent; private final List systemClasses; @@ -85,7 +106,7 @@ public ApplicationClassLoader(URL[] urls, ClassLoader parent, } // if the caller-specified system classes are null or empty, use the default this.systemClasses = (systemClasses == null || systemClasses.isEmpty()) ? - Arrays.asList(StringUtils.getTrimmedStrings(DEFAULT_SYSTEM_CLASSES)) : + Arrays.asList(StringUtils.getTrimmedStrings(SYSTEM_CLASSES_DEFAULT)) : systemClasses; LOG.info("system classes: " + this.systemClasses); } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/org.apache.hadoop.application-classloader.properties b/hadoop-common-project/hadoop-common/src/main/resources/org.apache.hadoop.application-classloader.properties new file mode 100644 index 0000000000..2264920ed5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/resources/org.apache.hadoop.application-classloader.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# contains key properties for setting up the application classloader +system.classes.default=java.,\ + javax.accessibility.,\ + javax.activation.,\ + javax.activity.,\ + javax.annotation.,\ + javax.annotation.processing.,\ + javax.crypto.,\ + javax.imageio.,\ + javax.jws.,\ + javax.lang.model.,\ + -javax.management.j2ee.,\ + javax.management.,\ + javax.naming.,\ + javax.net.,\ + javax.print.,\ + javax.rmi.,\ + javax.script.,\ + -javax.security.auth.message.,\ + javax.security.auth.,\ + javax.security.cert.,\ + javax.security.sasl.,\ + javax.sound.,\ + javax.sql.,\ + javax.swing.,\ + javax.tools.,\ + javax.transaction.,\ + -javax.xml.registry.,\ + -javax.xml.rpc.,\ + javax.xml.,\ + org.w3c.dom.,\ + org.xml.sax.,\ + org.apache.commons.logging.,\ + org.apache.log4j.,\ + org.apache.hadoop.,\ + core-default.xml,\ + hdfs-default.xml,\ + mapred-default.xml,\ + yarn-default.xml diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java index 9e279689a4..f592d0400a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java @@ -131,7 +131,7 @@ public void testClientClassLoader() throws Throwable { String thirdCls = ClassLoaderCheckThird.class.getName(); String systemClasses = "-" + mainCls + "," + "-" + thirdCls + "," + - ApplicationClassLoader.DEFAULT_SYSTEM_CLASSES; + ApplicationClassLoader.SYSTEM_CLASSES_DEFAULT; when(runJar.getSystemClasses()).thenReturn(systemClasses); // create the test jar diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index 02a59e73e2..96b4e84b55 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -498,7 +498,7 @@ public void testTaskStateUI() { private static final String[] SYS_CLASSES = new String[] { "/java/fake/Klass", - "/javax/fake/Klass", + "/javax/management/fake/Klass", "/org/apache/commons/logging/fake/Klass", "/org/apache/log4j/fake/Klass", "/org/apache/hadoop/fake/Klass" @@ -515,7 +515,7 @@ public void testTaskStateUI() { public void testSystemClasses() { final List systemClasses = Arrays.asList(StringUtils.getTrimmedStrings( - ApplicationClassLoader.DEFAULT_SYSTEM_CLASSES)); + ApplicationClassLoader.SYSTEM_CLASSES_DEFAULT)); for (String defaultXml : DEFAULT_XMLS) { assertTrue(defaultXml + " must be system resource", ApplicationClassLoader.isSystemClass(defaultXml, systemClasses)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java index 5699600acc..e27168d24f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java @@ -242,7 +242,7 @@ private void testJobClassloader(boolean useCustomClasses) throws IOException, // to test AM loading user classes such as output format class, we want // to blacklist them from the system classes (they need to be prepended // as the first match wins) - String systemClasses = ApplicationClassLoader.DEFAULT_SYSTEM_CLASSES; + String systemClasses = ApplicationClassLoader.SYSTEM_CLASSES_DEFAULT; // exclude the custom classes from system classes systemClasses = "-" + CustomOutputFormat.class.getName() + ",-" + CustomSpeculator.class.getName() + "," +