diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java index 190d988022..0b942b0ccd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java @@ -27,6 +27,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Set; +import java.util.Collections; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -81,7 +83,7 @@ public class LocalContainerLauncher extends AbstractService implements private static final Log LOG = LogFactory.getLog(LocalContainerLauncher.class); private FileContext curFC = null; - private final HashSet localizedFiles; + private Set localizedFiles = new HashSet(); private final AppContext context; private final TaskUmbilicalProtocol umbilical; private final ClassLoader jobClassLoader; @@ -121,9 +123,12 @@ public LocalContainerLauncher(AppContext context, // users who do that get what they deserve (and will have to disable // uberization in order to run correctly). File[] curLocalFiles = curDir.listFiles(); - localizedFiles = new HashSet(curLocalFiles.length); - for (int j = 0; j < curLocalFiles.length; ++j) { - localizedFiles.add(curLocalFiles[j]); + if (curLocalFiles != null) { + HashSet lf = new HashSet(curLocalFiles.length); + for (int j = 0; j < curLocalFiles.length; ++j) { + lf.add(curLocalFiles[j]); + } + localizedFiles = Collections.unmodifiableSet(lf); } // Relocalization note/future FIXME (per chrisdo, 20110315): At moment, @@ -521,26 +526,29 @@ private void runSubtask(org.apache.hadoop.mapred.Task task, */ private void relocalize() { File[] curLocalFiles = curDir.listFiles(); - for (int j = 0; j < curLocalFiles.length; ++j) { - if (!localizedFiles.contains(curLocalFiles[j])) { - // found one that wasn't there before: delete it - boolean deleted = false; - try { - if (curFC != null) { - // this is recursive, unlike File delete(): - deleted = curFC.delete(new Path(curLocalFiles[j].getName()),true); + if (curLocalFiles != null) { + for (int j = 0; j < curLocalFiles.length; ++j) { + if (!localizedFiles.contains(curLocalFiles[j])) { + // found one that wasn't there before: delete it + boolean deleted = false; + try { + if (curFC != null) { + // this is recursive, unlike File delete(): + deleted = + curFC.delete(new Path(curLocalFiles[j].getName()), true); + } + } catch (IOException e) { + deleted = false; + } + if (!deleted) { + LOG.warn("Unable to delete unexpected local file/dir " + + curLocalFiles[j].getName() + + ": insufficient permissions?"); } - } catch (IOException e) { - deleted = false; - } - if (!deleted) { - LOG.warn("Unable to delete unexpected local file/dir " - + curLocalFiles[j].getName() + ": insufficient permissions?"); } } } } - } // end EventHandler /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index bb7abb6407..ba7f929134 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -572,13 +572,15 @@ protected AMPreemptionPolicy createPreemptionPolicy(Configuration conf) { private boolean isJobNamePatternMatch(JobConf conf, String jobTempDir) { // Matched staging files should be preserved after job is finished. if (conf.getKeepTaskFilesPattern() != null && jobTempDir != null) { - String jobFileName = Paths.get(jobTempDir).getFileName().toString(); - Pattern pattern = Pattern.compile(conf.getKeepTaskFilesPattern()); - Matcher matcher = pattern.matcher(jobFileName); - return matcher.find(); - } else { - return false; + java.nio.file.Path pathName = Paths.get(jobTempDir).getFileName(); + if (pathName != null) { + String jobFileName = pathName.toString(); + Pattern pattern = Pattern.compile(conf.getKeepTaskFilesPattern()); + Matcher matcher = pattern.matcher(jobFileName); + return matcher.find(); + } } + return false; } private boolean isKeepFailedTaskFiles(JobConf conf) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java index 77a7f8a909..6e0e1c41d3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java @@ -98,7 +98,7 @@ public int compareTo(JVMId that) { int jobComp = this.jobId.compareTo(that.jobId); if(jobComp == 0) { if(this.isMap == that.isMap) { - return Long.valueOf(this.jvmId).compareTo(that.jvmId); + return Long.compare(this.jvmId, that.jvmId); } else { return this.isMap ? -1 : 1; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java index 4df9be44fe..a5552f88a0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java @@ -34,12 +34,20 @@ public enum Operation { KILL_TASK(QueueACL.ADMINISTER_JOBS, JobACL.MODIFY_JOB), SET_JOB_PRIORITY(QueueACL.ADMINISTER_JOBS, JobACL.MODIFY_JOB), SUBMIT_JOB(QueueACL.SUBMIT_JOB, null); - - public QueueACL qACLNeeded; - public JobACL jobACLNeeded; + + private final QueueACL qACLNeeded; + private final JobACL jobACLNeeded; Operation(QueueACL qACL, JobACL jobACL) { this.qACLNeeded = qACL; this.jobACLNeeded = jobACL; } + + public QueueACL getqACLNeeded() { + return qACLNeeded; + } + + public JobACL getJobACLNeeded() { + return jobACLNeeded; + } } \ No newline at end of file diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index bd8d9c220a..6b98e47055 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -407,17 +407,12 @@ synchronized void moveToDone() throws IOException { } JobId jobId = jobIndexInfo.getJobId(); - List paths = new ArrayList(2); if (historyFile == null) { LOG.info("No file for job-history with " + jobId + " found in cache!"); - } else { - paths.add(historyFile); } if (confFile == null) { LOG.info("No file for jobConf with " + jobId + " found in cache!"); - } else { - paths.add(confFile); } if (summaryFile == null || !intermediateDoneDirFc.util().exists( diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java index a2db9d1bac..60b36387b8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java @@ -67,8 +67,12 @@ private static void parseLine(final String line, Map private void parse(File f, Map> sums) throws IOException { if (f.isDirectory()) { println("Process directory " + f); - for(File child : f.listFiles()) - parse(child, sums); + File[] files = f.listFiles(); + if (files != null) { + for(File child : files) { + parse(child, sums); + } + } } else if (f.getName().endsWith(".txt")) { println("Parse file " + f); final Map> m = new TreeMap>();