MAPREDUCE-6881. Fix warnings from Spotbugs in hadoop-mapreduce. Contributed by Weiwei Yang.
This commit is contained in:
parent
28eb2aabeb
commit
3ed3062fe3
@ -27,6 +27,8 @@
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.Collections;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
@ -81,7 +83,7 @@ public class LocalContainerLauncher extends AbstractService implements
|
||||
private static final Log LOG = LogFactory.getLog(LocalContainerLauncher.class);
|
||||
|
||||
private FileContext curFC = null;
|
||||
private final HashSet<File> localizedFiles;
|
||||
private Set<File> localizedFiles = new HashSet<File>();
|
||||
private final AppContext context;
|
||||
private final TaskUmbilicalProtocol umbilical;
|
||||
private final ClassLoader jobClassLoader;
|
||||
@ -121,9 +123,12 @@ public LocalContainerLauncher(AppContext context,
|
||||
// users who do that get what they deserve (and will have to disable
|
||||
// uberization in order to run correctly).
|
||||
File[] curLocalFiles = curDir.listFiles();
|
||||
localizedFiles = new HashSet<File>(curLocalFiles.length);
|
||||
for (int j = 0; j < curLocalFiles.length; ++j) {
|
||||
localizedFiles.add(curLocalFiles[j]);
|
||||
if (curLocalFiles != null) {
|
||||
HashSet<File> lf = new HashSet<File>(curLocalFiles.length);
|
||||
for (int j = 0; j < curLocalFiles.length; ++j) {
|
||||
lf.add(curLocalFiles[j]);
|
||||
}
|
||||
localizedFiles = Collections.unmodifiableSet(lf);
|
||||
}
|
||||
|
||||
// Relocalization note/future FIXME (per chrisdo, 20110315): At moment,
|
||||
@ -521,26 +526,29 @@ private void runSubtask(org.apache.hadoop.mapred.Task task,
|
||||
*/
|
||||
private void relocalize() {
|
||||
File[] curLocalFiles = curDir.listFiles();
|
||||
for (int j = 0; j < curLocalFiles.length; ++j) {
|
||||
if (!localizedFiles.contains(curLocalFiles[j])) {
|
||||
// found one that wasn't there before: delete it
|
||||
boolean deleted = false;
|
||||
try {
|
||||
if (curFC != null) {
|
||||
// this is recursive, unlike File delete():
|
||||
deleted = curFC.delete(new Path(curLocalFiles[j].getName()),true);
|
||||
if (curLocalFiles != null) {
|
||||
for (int j = 0; j < curLocalFiles.length; ++j) {
|
||||
if (!localizedFiles.contains(curLocalFiles[j])) {
|
||||
// found one that wasn't there before: delete it
|
||||
boolean deleted = false;
|
||||
try {
|
||||
if (curFC != null) {
|
||||
// this is recursive, unlike File delete():
|
||||
deleted =
|
||||
curFC.delete(new Path(curLocalFiles[j].getName()), true);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
deleted = false;
|
||||
}
|
||||
if (!deleted) {
|
||||
LOG.warn("Unable to delete unexpected local file/dir "
|
||||
+ curLocalFiles[j].getName()
|
||||
+ ": insufficient permissions?");
|
||||
}
|
||||
} catch (IOException e) {
|
||||
deleted = false;
|
||||
}
|
||||
if (!deleted) {
|
||||
LOG.warn("Unable to delete unexpected local file/dir "
|
||||
+ curLocalFiles[j].getName() + ": insufficient permissions?");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // end EventHandler
|
||||
|
||||
/**
|
||||
|
@ -572,13 +572,15 @@ protected AMPreemptionPolicy createPreemptionPolicy(Configuration conf) {
|
||||
private boolean isJobNamePatternMatch(JobConf conf, String jobTempDir) {
|
||||
// Matched staging files should be preserved after job is finished.
|
||||
if (conf.getKeepTaskFilesPattern() != null && jobTempDir != null) {
|
||||
String jobFileName = Paths.get(jobTempDir).getFileName().toString();
|
||||
Pattern pattern = Pattern.compile(conf.getKeepTaskFilesPattern());
|
||||
Matcher matcher = pattern.matcher(jobFileName);
|
||||
return matcher.find();
|
||||
} else {
|
||||
return false;
|
||||
java.nio.file.Path pathName = Paths.get(jobTempDir).getFileName();
|
||||
if (pathName != null) {
|
||||
String jobFileName = pathName.toString();
|
||||
Pattern pattern = Pattern.compile(conf.getKeepTaskFilesPattern());
|
||||
Matcher matcher = pattern.matcher(jobFileName);
|
||||
return matcher.find();
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean isKeepFailedTaskFiles(JobConf conf) {
|
||||
|
@ -98,7 +98,7 @@ public int compareTo(JVMId that) {
|
||||
int jobComp = this.jobId.compareTo(that.jobId);
|
||||
if(jobComp == 0) {
|
||||
if(this.isMap == that.isMap) {
|
||||
return Long.valueOf(this.jvmId).compareTo(that.jvmId);
|
||||
return Long.compare(this.jvmId, that.jvmId);
|
||||
} else {
|
||||
return this.isMap ? -1 : 1;
|
||||
}
|
||||
|
@ -35,11 +35,19 @@ public enum Operation {
|
||||
SET_JOB_PRIORITY(QueueACL.ADMINISTER_JOBS, JobACL.MODIFY_JOB),
|
||||
SUBMIT_JOB(QueueACL.SUBMIT_JOB, null);
|
||||
|
||||
public QueueACL qACLNeeded;
|
||||
public JobACL jobACLNeeded;
|
||||
private final QueueACL qACLNeeded;
|
||||
private final JobACL jobACLNeeded;
|
||||
|
||||
Operation(QueueACL qACL, JobACL jobACL) {
|
||||
this.qACLNeeded = qACL;
|
||||
this.jobACLNeeded = jobACL;
|
||||
}
|
||||
|
||||
public QueueACL getqACLNeeded() {
|
||||
return qACLNeeded;
|
||||
}
|
||||
|
||||
public JobACL getJobACLNeeded() {
|
||||
return jobACLNeeded;
|
||||
}
|
||||
}
|
@ -407,17 +407,12 @@ synchronized void moveToDone() throws IOException {
|
||||
}
|
||||
JobId jobId = jobIndexInfo.getJobId();
|
||||
|
||||
List<Path> paths = new ArrayList<Path>(2);
|
||||
if (historyFile == null) {
|
||||
LOG.info("No file for job-history with " + jobId + " found in cache!");
|
||||
} else {
|
||||
paths.add(historyFile);
|
||||
}
|
||||
|
||||
if (confFile == null) {
|
||||
LOG.info("No file for jobConf with " + jobId + " found in cache!");
|
||||
} else {
|
||||
paths.add(confFile);
|
||||
}
|
||||
|
||||
if (summaryFile == null || !intermediateDoneDirFc.util().exists(
|
||||
|
@ -67,8 +67,12 @@ private static void parseLine(final String line, Map<Parameter, List<TaskResult>
|
||||
private void parse(File f, Map<Parameter, List<TaskResult>> sums) throws IOException {
|
||||
if (f.isDirectory()) {
|
||||
println("Process directory " + f);
|
||||
for(File child : f.listFiles())
|
||||
parse(child, sums);
|
||||
File[] files = f.listFiles();
|
||||
if (files != null) {
|
||||
for(File child : files) {
|
||||
parse(child, sums);
|
||||
}
|
||||
}
|
||||
} else if (f.getName().endsWith(".txt")) {
|
||||
println("Parse file " + f);
|
||||
final Map<Parameter, List<TaskResult>> m = new TreeMap<Parameter, List<TaskResult>>();
|
||||
|
Loading…
Reference in New Issue
Block a user