diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 27f1328382..ce4e3242cf 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -423,6 +423,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8108. Move method getHostPortString() from NameNode to NetUtils. (Brandon Li via jitendra) + HADOOP-8288. Remove references of mapred.child.ulimit etc. since they are + not being used any more (Ravi Prakash via bobby) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index 1b35ccfc39..615956bdf7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -81,64 +81,6 @@ public static String[] getGET_PERMISSION_COMMAND() { /** If or not script timed out*/ private AtomicBoolean timedOut; - /** a Unix command to get ulimit of a process. */ - public static final String ULIMIT_COMMAND = "ulimit"; - - /** - * Get the Unix command for setting the maximum virtual memory available - * to a given child process. This is only relevant when we are forking a - * process from within the Mapper or the Reducer implementations. - * Also see Hadoop Pipes and Hadoop Streaming. - * - * It also checks to ensure that we are running on a *nix platform else - * (e.g. in Cygwin/Windows) it returns null. - * @param memoryLimit virtual memory limit - * @return a String[] with the ulimit command arguments or - * null if we are running on a non *nix platform or - * if the limit is unspecified. - */ - public static String[] getUlimitMemoryCommand(int memoryLimit) { - // ulimit isn't supported on Windows - if (WINDOWS) { - return null; - } - - return new String[] {ULIMIT_COMMAND, "-v", String.valueOf(memoryLimit)}; - } - - /** - * Get the Unix command for setting the maximum virtual memory available - * to a given child process. This is only relevant when we are forking a - * process from within the Mapper or the Reducer implementations. - * see also Hadoop Pipes and Streaming. - * - * It also checks to ensure that we are running on a *nix platform else - * (e.g. in Cygwin/Windows) it returns null. - * @param conf configuration - * @return a String[] with the ulimit command arguments or - * null if we are running on a non *nix platform or - * if the limit is unspecified. - * @deprecated Use {@link #getUlimitMemoryCommand(int)} - */ - @Deprecated - public static String[] getUlimitMemoryCommand(Configuration conf) { - // ulimit isn't supported on Windows - if (WINDOWS) { - return null; - } - - // get the memory limit from the configuration - String ulimit = conf.get("mapred.child.ulimit"); - if (ulimit == null) { - return null; - } - - // Parse it to ensure it is legal/sane - int memoryLimit = Integer.valueOf(ulimit); - - return getUlimitMemoryCommand(memoryLimit); - } - /** Set to true on Windows platforms */ public static final boolean WINDOWS /* borrowed from Path.WINDOWS */ = System.getProperty("os.name").startsWith("Windows"); diff --git a/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml b/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml index 91682bbb33..f1f36982e9 100644 --- a/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml +++ b/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml @@ -108,11 +108,6 @@ -server -Xmx640m -Djava.net.preferIPv4Stack=true - - mapred.child.ulimit - 8388608 - - mapred.job.tracker.persist.jobstatus.active true diff --git a/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm index 05b160fbe9..c391587db1 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm @@ -314,8 +314,6 @@ Deprecated Properties *---+---+ |mapred.map.child.log.level | mapreduce.map.log.level *---+---+ -|mapred.map.child.ulimit | mapreduce.map.ulimit -*---+---+ |mapred.map.max.attempts | mapreduce.map.maxattempts *---+---+ |mapred.map.output.compression.codec | mapreduce.map.output.compress.codec @@ -378,8 +376,6 @@ Deprecated Properties *---+---+ |mapred.reduce.child.log.level | mapreduce.reduce.log.level *---+---+ -|mapred.reduce.child.ulimit | mapreduce.reduce.ulimit -*---+---+ |mapred.reduce.max.attempts | mapreduce.reduce.maxattempts *---+---+ |mapred.reduce.parallel.copies | mapreduce.reduce.shuffle.parallelcopies diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUlimit.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUlimit.java deleted file mode 100644 index 89b4d49a23..0000000000 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUlimit.java +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.streaming; - -import java.io.*; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.MapReduceTestUtil; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.util.StringUtils; - -import org.junit.Ignore; -import org.junit.Test; -import static org.junit.Assert.*; - -/** - * This tests the setting of memory limit for streaming processes. - * This will launch a streaming app which will allocate 10MB memory. - * First, program is launched with sufficient memory. And test expects - * it to succeed. Then program is launched with insufficient memory and - * is expected to be a failure. - */ -public class TestUlimit { - String input = "the dummy input"; - Path inputPath = new Path("target/testing/in"); - Path outputPath = new Path("target/testing/out"); - String map = null; - MiniDFSCluster dfs = null; - MiniMRCluster mr = null; - FileSystem fs = null; - private static String SET_MEMORY_LIMIT = "786432"; // 768MB - - String[] genArgs(String memLimit) { - String strNamenode = "fs.default.name=" + mr.createJobConf().get("fs.default.name"); - return new String[] { - "-input", inputPath.toString(), - "-output", outputPath.toString(), - "-mapper", map, - "-reducer", "org.apache.hadoop.mapred.lib.IdentityReducer", - "-numReduceTasks", "0", - "-jobconf", MRJobConfig.NUM_MAPS + "=1", - "-jobconf", JobConf.MAPRED_MAP_TASK_ULIMIT + "=" + memLimit, - "-jobconf", strNamenode, - "-jobconf", "stream.tmpdir=" + - System.getProperty("test.build.data","/tmp"), - "-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR, - "-jobconf", "mapreduce.framework.name=yarn" - }; - } - - /** - * This tests the setting of memory limit for streaming processes. - * This will launch a streaming app which will allocate 10MB memory. - * First, program is launched with sufficient memory. And test expects - * it to succeed. Then program is launched with insufficient memory and - * is expected to be a failure. - */ - @Test - @Ignore - public void testCommandLine() { - if (UtilTest.isCygwin()) { - return; - } - try { - final int numSlaves = 2; - Configuration conf = new Configuration(); - dfs = new MiniDFSCluster(conf, numSlaves, true, null); - fs = dfs.getFileSystem(); - - mr = new MiniMRCluster(numSlaves, fs.getUri().toString(), 1); - writeInputFile(fs, inputPath); - map = UtilTest.makeJavaCommand(UlimitApp.class, new String[]{}); - runProgram(SET_MEMORY_LIMIT); - fs.delete(outputPath, true); - assertFalse("output not cleaned up", fs.exists(outputPath)); - mr.waitUntilIdle(); - } catch(IOException e) { - fail(StringUtils.stringifyException(e)); - } finally { - if (mr != null) { - mr.shutdown(); - } - if (dfs != null) { - dfs.shutdown(); - } - } - } - - private void writeInputFile(FileSystem fs, Path dir) throws IOException { - DataOutputStream out = fs.create(new Path(dir, "part0")); - out.writeBytes(input); - out.close(); - } - - /** - * Runs the streaming program. and asserts the result of the program. - * @param memLimit memory limit to set for mapred child. - * @param result Expected result - * @throws IOException - */ - private void runProgram(String memLimit) throws IOException { - boolean mayExit = false; - StreamJob job = new StreamJob(genArgs(memLimit), mayExit); - job.go(); - String output = MapReduceTestUtil.readOutput(outputPath, - mr.createJobConf()); - assertEquals("output is wrong", SET_MEMORY_LIMIT, - output.trim()); - } - - public static void main(String[]args) throws Exception - { - new TestUlimit().testCommandLine(); - } - -} diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UlimitApp.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UlimitApp.java deleted file mode 100644 index 92d40f611d..0000000000 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UlimitApp.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.streaming; - -import java.io.*; - -/** - * The UlimitApp discards the input - * and exec's ulimit -v to know the ulimit value. - * And writes the output to the standard out. - * @see {@link TestUlimit} - */ -public class UlimitApp { - public static void main(String args[]) throws IOException{ - BufferedReader in = new BufferedReader( - new InputStreamReader(System.in)); - String line = null; - while ((line = in.readLine()) != null) {} - - Process process = Runtime.getRuntime().exec(new String[]{ - "bash", "-c", "ulimit -v"}); - InputStream is = process.getInputStream(); - InputStreamReader isr = new InputStreamReader(is); - BufferedReader br = new BufferedReader(isr); - while ((line = br.readLine()) != null) { - System.out.println(line); - } - } -}