From 72e9ec7a451d9fd1379e9d5ec0e8bfb7b31db500 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Sat, 3 Dec 2011 00:16:30 +0000 Subject: [PATCH] MAPREDUCE-3479. JobClient#getJob cannot find local jobs. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1209791 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 + .../hadoop/mapred/TestJobClientGetJob.java | 62 +++++++++++++++++++ .../org/apache/hadoop/mapred/JobClient.java | 4 ++ .../java/org/apache/hadoop/mapreduce/Job.java | 5 ++ 4 files changed, 73 insertions(+) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 00c1d05804..bf5683d372 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -226,6 +226,8 @@ Release 0.23.1 - Unreleased MAPREDUCE-3453. RM web ui application details page shows RM cluster about information. (Jonathan Eagles via sseth) + MAPREDUCE-3479. JobClient#getJob cannot find local jobs. (tomwhite) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java new file mode 100644 index 0000000000..5ba5f25a63 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import static junit.framework.Assert.assertNotNull; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +public class TestJobClientGetJob { + + private static Path TEST_ROOT_DIR = + new Path(System.getProperty("test.build.data","/tmp")); + + private Path createTempFile(String filename, String contents) + throws IOException { + Path path = new Path(TEST_ROOT_DIR, filename); + Configuration conf = new Configuration(); + FSDataOutputStream os = FileSystem.getLocal(conf).create(path); + os.writeBytes(contents); + os.close(); + return path; + } + + @SuppressWarnings("deprecation") + @Test + public void testGetRunningJobFromJobClient() throws Exception { + JobConf conf = new JobConf(); + conf.set("mapreduce.framework.name", "local"); + FileInputFormat.addInputPath(conf, createTempFile("in", "hello")); + FileOutputFormat.setOutputPath(conf, + new Path(TEST_ROOT_DIR, getClass().getSimpleName())); + JobClient jc = new JobClient(conf); + RunningJob runningJob = jc.submitJob(conf); + assertNotNull("Running job", runningJob); + // Check that the running job can be retrieved by ID + RunningJob newRunningJob = jc.getJob(runningJob.getID()); + assertNotNull("New running job", newRunningJob); + } + +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java index fc65d46812..1b0b453ddd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java @@ -584,6 +584,10 @@ public Job run() throws IOException, ClassNotFoundException, return job; } }); + // update our Cluster instance with the one created by Job for submission + // (we can't pass our Cluster instance to Job, since Job wraps the config + // instance, and the two configs would then diverge) + cluster = job.getCluster(); return new NetworkedJob(job); } catch (InterruptedException ie) { throw new IOException("interrupted", ie); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java index c6b5adeb43..5e92baa8b6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java @@ -436,6 +436,11 @@ public boolean isRetired() throws IOException, InterruptedException { updateStatus(); return status.isRetired(); } + + @Private + public Cluster getCluster() { + return cluster; + } /** Only for mocks in unit tests. */ @Private