From 1c15670d531945e3cae856af35bc0e52567c2ea5 Mon Sep 17 00:00:00 2001 From: Mahadev Konar Date: Wed, 7 Sep 2011 21:18:09 +0000 Subject: [PATCH] MAPREDUCE-2948. Hadoop streaming test failure, post MR-2767 (mahadev) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1166402 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 + .../TestStreamingAsDifferentUser.java | 178 ------------------ 2 files changed, 2 insertions(+), 178 deletions(-) delete mode 100644 hadoop-mapreduce-project/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingAsDifferentUser.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a5b5a08e09..58fa824442 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -1219,6 +1219,8 @@ Release 0.23.0 - Unreleased MAPREDUCE-2938. Log application submission failure in CapacityScheduler. (acmurthy) + MAPREDUCE-2948. Hadoop streaming test failure, post MR-2767 (mahadev) + Release 0.22.0 - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingAsDifferentUser.java b/hadoop-mapreduce-project/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingAsDifferentUser.java deleted file mode 100644 index 5843269d66..0000000000 --- a/hadoop-mapreduce-project/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingAsDifferentUser.java +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.streaming; - -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.security.PrivilegedExceptionAction; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.mapred.ClusterWithLinuxTaskController; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.security.Groups; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.Shell; -import org.apache.hadoop.util.ToolRunner; - -/** - * Test Streaming with LinuxTaskController running the jobs as a user different - * from the user running the cluster. See {@link ClusterWithLinuxTaskController} - */ -public class TestStreamingAsDifferentUser extends - ClusterWithLinuxTaskController { - - private Path inputPath = new Path("input"); - private Path outputPath = new Path("output"); - private String input = "roses.are.red\nviolets.are.blue\nbunnies.are.pink\n"; - private String map = - UtilTest.makeJavaCommand(TrApp.class, new String[] { ".", "\\n" }); - private String reduce = - UtilTest.makeJavaCommand(UniqApp.class, new String[] { "R" }); - - public void testStreaming() - throws Exception { - if (!shouldRun()) { - return; - } - startCluster(); - final JobConf myConf = getClusterConf(); - jobOwner.doAs(new PrivilegedExceptionAction() { - public Void run() throws IOException{ - - FileSystem inFs = inputPath.getFileSystem(myConf); - FileSystem outFs = outputPath.getFileSystem(myConf); - outFs.delete(outputPath, true); - if (!inFs.mkdirs(inputPath)) { - throw new IOException("Mkdirs failed to create " + inFs.toString()); - } - DataOutputStream file = inFs.create(new Path(inputPath, "part-0")); - file.writeBytes(input); - file.close(); - final String[] args = - new String[] { "-input", inputPath.makeQualified(inFs).toString(), - "-output", outputPath.makeQualified(outFs).toString(), "-mapper", - map, "-reducer", reduce, "-jobconf", - "mapreduce.task.files.preserve.failedtasks=true", "-jobconf", - "stream.tmpdir=" + System.getProperty("test.build.data", "/tmp") }; - - StreamJob streamJob = new StreamJob(args, true); - streamJob.setConf(myConf); - assertTrue("Job has not succeeded", streamJob.go() == 0); - assertOwnerShip(outputPath); - return null; - } - }); - } - - /** - * Verify if the permissions of distcache dir contents are valid once the job - * is finished - */ - public void testStreamingWithDistCache() - throws Exception { - if (!shouldRun()) { - return; - } - startCluster(); - final String[] localDirs = mrCluster.getTaskTrackerLocalDirs(0); - final JobConf myConf = getClusterConf(); - - // create file that will go into public distributed cache - File publicFile = new File(System.getProperty( - "test.build.data", "/tmp"), "publicFile"); - FileOutputStream fstream = new FileOutputStream(publicFile); - fstream.write("public file contents".getBytes()); - fstream.close(); - - // put the file(that should go into public dist cache) in dfs and set - // read and exe permissions for others - FileSystem dfs = dfsCluster.getFileSystem(); - dfs.setPermission(new Path(dfs.getDefaultUri(myConf).toString() + "/tmp"), - new FsPermission((short)0755)); - final String publicCacheFile = dfs.getDefaultUri(myConf).toString() - + "/tmp/publicFile"; - dfs.copyFromLocalFile(new Path(publicFile.getAbsolutePath()), - new Path(publicCacheFile)); - dfs.setPermission(new Path(publicCacheFile), new FsPermission((short)0755)); - final String taskTrackerUser - = UserGroupInformation.getCurrentUser().getShortUserName(); - - jobOwner.doAs(new PrivilegedExceptionAction() { - public Void run() throws Exception{ - - FileSystem inFs = inputPath.getFileSystem(myConf); - FileSystem outFs = outputPath.getFileSystem(myConf); - outFs.delete(outputPath, true); - if (!inFs.mkdirs(inputPath)) { - throw new IOException("Mkdirs failed to create " + inFs.toString()); - } - - // create input file - DataOutputStream file = inFs.create(new Path(inputPath, "part-0")); - file.writeBytes(input); - file.close(); - - // Create file that will be passed using -files option. - // This is private dist cache file - File privateFile = new File(System.getProperty( - "test.build.data", "/tmp"), "test.sh"); - privateFile.createNewFile(); - - String[] args = - new String[] { - "-files", privateFile.toString() + "," + publicCacheFile, - "-Dmapreduce.task.files.preserve.failedtasks=true", - "-Dstream.tmpdir=" + System.getProperty("test.build.data", "/tmp"), - "-input", inputPath.makeQualified(inFs).toString(), - "-output", outputPath.makeQualified(outFs).toString(), - "-mapper", "pwd", - "-reducer", StreamJob.REDUCE_NONE - }; - StreamJob streamJob = new StreamJob(); - streamJob.setConf(myConf); - - assertTrue("Job failed", ToolRunner.run(streamJob, args)==0); - - // validate private cache files' permissions - checkPermissionsOnPrivateDistCache(localDirs, - jobOwner.getShortUserName(), taskTrackerUser, - taskTrackerSpecialGroup); - - // check the file is present even after the job is over. - // work directory symlink cleanup should not have removed the target - // files. - checkPresenceOfPrivateDistCacheFiles(localDirs, - jobOwner.getShortUserName(), new String[] {"test.sh"}); - - // validate private cache files' permissions - checkPermissionsOnPublicDistCache(FileSystem.getLocal(myConf), - localDirs, taskTrackerUser, taskTrackerPrimaryGroup); - - checkPresenceOfPublicDistCacheFiles(localDirs, - new String[] {"publicFile"}); - assertOwnerShip(outputPath); - return null; - } - }); - } -}