From d14e26b31fe46fb47a8e99a212c70016fd15a4d9 Mon Sep 17 00:00:00 2001 From: Johan Gustavsson Date: Mon, 28 May 2018 17:29:59 +0900 Subject: [PATCH] HADOOP-15477. Make unjar in RunJar overrideable Signed-off-by: Akira Ajisaka --- .../java/org/apache/hadoop/util/RunJar.java | 17 +++++++-- .../org/apache/hadoop/util/TestRunJar.java | 37 ++++++++++++++++++- .../apache/hadoop/streaming/StreamJob.java | 4 +- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java index 9dd770c43a..f1b643c4b0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java @@ -76,7 +76,11 @@ public class RunJar { */ public static final String HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES = "HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES"; - + /** + * Environment key for disabling unjar in client code. + */ + public static final String HADOOP_CLIENT_SKIP_UNJAR = + "HADOOP_CLIENT_SKIP_UNJAR"; /** * Buffer size for copy the content of compressed file to new file. */ @@ -93,7 +97,7 @@ public class RunJar { * @throws IOException if an I/O error has occurred or toDir * cannot be created and does not already exist */ - public static void unJar(File jarFile, File toDir) throws IOException { + public void unJar(File jarFile, File toDir) throws IOException { unJar(jarFile, toDir, MATCH_ANY); } @@ -292,8 +296,9 @@ public void run() { } }, SHUTDOWN_HOOK_PRIORITY); - - unJar(file, workDir); + if (!skipUnjar()) { + unJar(file, workDir); + } ClassLoader loader = createClassLoader(file, workDir); @@ -364,6 +369,10 @@ boolean useClientClassLoader() { return Boolean.parseBoolean(System.getenv(HADOOP_USE_CLIENT_CLASSLOADER)); } + boolean skipUnjar() { + return Boolean.parseBoolean(System.getenv(HADOOP_CLIENT_SKIP_UNJAR)); + } + String getHadoopClasspath() { return System.getenv(HADOOP_CLASSPATH); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java index 19485d69e8..ea07b970a2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java @@ -17,10 +17,14 @@ */ package org.apache.hadoop.util; +import static org.apache.hadoop.util.RunJar.MATCH_ANY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.File; @@ -99,7 +103,7 @@ public void testUnJar() throws Exception { // Unjar everything RunJar.unJar(new File(TEST_ROOT_DIR, TEST_JAR_NAME), - unjarDir); + unjarDir, MATCH_ANY); assertTrue("foobar unpacked", new File(unjarDir, TestRunJar.FOOBAR_TXT).exists()); assertTrue("foobaz unpacked", @@ -177,7 +181,7 @@ public void testUnJarDoesNotLooseLastModify() throws Exception { // Unjar everything RunJar.unJar(new File(TEST_ROOT_DIR, TEST_JAR_NAME), - unjarDir); + unjarDir, MATCH_ANY); String failureMessage = "Last modify time was lost during unJar"; assertEquals(failureMessage, MOCKED_NOW, new File(unjarDir, TestRunJar.FOOBAR_TXT).lastModified()); @@ -221,5 +225,34 @@ public void testClientClassLoader() throws Throwable { // run RunJar runJar.run(args); // it should not throw an exception + verify(runJar, times(1)).unJar(any(File.class), any(File.class)); + } + + @Test + public void testClientClassLoaderSkipUnjar() throws Throwable { + RunJar runJar = spy(new RunJar()); + // enable the client classloader + when(runJar.useClientClassLoader()).thenReturn(true); + // set the system classes and blacklist the test main class and the test + // third class so they can be loaded by the application classloader + String mainCls = ClassLoaderCheckMain.class.getName(); + String thirdCls = ClassLoaderCheckThird.class.getName(); + String systemClasses = "-" + mainCls + "," + + "-" + thirdCls + "," + + ApplicationClassLoader.SYSTEM_CLASSES_DEFAULT; + when(runJar.getSystemClasses()).thenReturn(systemClasses); + + // create the test jar + File testJar = JarFinder.makeClassLoaderTestJar(this.getClass(), + TEST_ROOT_DIR, TEST_JAR_2_NAME, BUFF_SIZE, mainCls, thirdCls); + // form the args + String[] args = new String[3]; + args[0] = testJar.getAbsolutePath(); + args[1] = mainCls; + when(runJar.skipUnjar()).thenReturn(true); + // run RunJar + runJar.run(args); + // it should not throw an exception + verify(runJar, times(0)).unJar(any(File.class), any(File.class)); } } \ No newline at end of file diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java index 9b09729741..1fe8710f01 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java @@ -72,6 +72,8 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; +import static org.apache.hadoop.util.RunJar.MATCH_ANY; + /** All the client-side work happens here. * (Jar packaging, MapRed job submission and monitoring) */ @@ -1006,7 +1008,7 @@ public int submitAndMonitorJob() throws IOException { if (jar_ != null && isLocalHadoop()) { // getAbs became required when shell and subvm have different working dirs... File wd = new File(".").getAbsoluteFile(); - RunJar.unJar(new File(jar_), wd); + RunJar.unJar(new File(jar_), wd, MATCH_ANY); } // if jobConf_ changes must recreate a JobClient