From a9808de0d9a73a99c10a3e4290ec20778fed4f24 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Tue, 8 May 2012 13:20:56 +0000 Subject: [PATCH] HADOOP-8341. Fix or filter findbugs issues in hadoop-tools (bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1335505 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 ++ .../apache/hadoop/tools/HadoopArchives.java | 2 +- .../java/org/apache/hadoop/tools/DistCp.java | 9 ++++-- .../org/apache/hadoop/tools/Logalyzer.java | 4 +-- .../dev-support/findbugs-exclude.xml | 31 +++++++++++++++++++ hadoop-tools/hadoop-rumen/pom.xml | 10 ++++++ .../tools/rumen/DeskewedJobTraceReader.java | 4 ++- .../tools/rumen/JobConfPropertyNames.java | 4 ++- .../tools/rumen/LoggedNetworkTopology.java | 4 ++- .../hadoop/tools/rumen/TraceBuilder.java | 3 +- .../WordListAnonymizerUtility.java | 2 +- .../tools/rumen/datatypes/NodeName.java | 12 ++----- .../dev-support/findbugs-exclude.xml | 30 ++++++++++++++++++ hadoop-tools/hadoop-streaming/pom.xml | 10 ++++++ .../apache/hadoop/streaming/StreamJob.java | 14 +++++---- 15 files changed, 114 insertions(+), 27 deletions(-) create mode 100644 hadoop-tools/hadoop-rumen/dev-support/findbugs-exclude.xml create mode 100644 hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 4e61a5bfd7..0cc39a58b4 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -545,6 +545,8 @@ Release 0.23.3 - UNRELEASED HADOOP-8327. distcpv2 and distcpv1 jars should not coexist (Dave Thompson via bobby) + HADOOP-8341. Fix or filter findbugs issues in hadoop-tools (bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java index 8a370a5eb4..98b3c9ca1e 100644 --- a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java +++ b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java @@ -117,7 +117,7 @@ public void setConf(Configuration conf) { // will when running the mapreduce job. String testJar = System.getProperty(TEST_HADOOP_ARCHIVES_JAR_PATH, null); if (testJar != null) { - ((JobConf)conf).setJar(testJar); + this.conf.setJar(testJar); } } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java index 05a581f43f..523609b532 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java @@ -136,10 +136,13 @@ public Job execute() throws Exception { Job job = null; try { - metaFolder = createMetaFolderPath(); - jobFS = metaFolder.getFileSystem(getConf()); + synchronized(this) { + //Don't cleanup while we are setting up. + metaFolder = createMetaFolderPath(); + jobFS = metaFolder.getFileSystem(getConf()); - job = createJob(); + job = createJob(); + } createInputFileListing(job); job.submit(); diff --git a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java index acc88ea1c1..4e7ac35b87 100644 --- a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java +++ b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java @@ -65,9 +65,9 @@ public class Logalyzer { // Constants private static Configuration fsConfig = new Configuration(); - public static String SORT_COLUMNS = + public static final String SORT_COLUMNS = "logalizer.logcomparator.sort.columns"; - public static String COLUMN_SEPARATOR = + public static final String COLUMN_SEPARATOR = "logalizer.logcomparator.column.separator"; static { diff --git a/hadoop-tools/hadoop-rumen/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-rumen/dev-support/findbugs-exclude.xml new file mode 100644 index 0000000000..93e4819486 --- /dev/null +++ b/hadoop-tools/hadoop-rumen/dev-support/findbugs-exclude.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml index 0c3d225738..8f5fddd577 100644 --- a/hadoop-tools/hadoop-rumen/pom.xml +++ b/hadoop-tools/hadoop-rumen/pom.xml @@ -90,6 +90,16 @@ + + org.codehaus.mojo + findbugs-maven-plugin + + true + true + ${basedir}/dev-support/findbugs-exclude.xml + Max + + org.apache.maven.plugins maven-antrun-plugin diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/DeskewedJobTraceReader.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/DeskewedJobTraceReader.java index ab6220ea84..e651ae8e05 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/DeskewedJobTraceReader.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/DeskewedJobTraceReader.java @@ -20,6 +20,7 @@ import java.io.Closeable; import java.io.IOException; +import java.io.Serializable; import java.util.Comparator; import java.util.Iterator; import java.util.PriorityQueue; @@ -59,7 +60,8 @@ public class DeskewedJobTraceReader implements Closeable { static final private Log LOG = LogFactory.getLog(DeskewedJobTraceReader.class); - static private class JobComparator implements Comparator { + static private class JobComparator implements Comparator, + Serializable { @Override public int compare(LoggedJob j1, LoggedJob j2) { return (j1.getSubmitTime() < j2.getSubmitTime()) ? -1 : (j1 diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java index 3e60a3627e..c1ba83ece4 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.tools.rumen; +import java.util.Arrays; + import org.apache.hadoop.mapreduce.MRJobConfig; public enum JobConfPropertyNames { @@ -33,6 +35,6 @@ public enum JobConfPropertyNames { } public String[] getCandidates() { - return candidates; + return Arrays.copyOf(candidates, candidates.length); } } diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java index 3d9caf0b35..23bbb98bb0 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.tools.rumen; +import java.io.Serializable; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -68,7 +69,8 @@ public void setUnknownAttribute(String attributeName, Object ignored) { * order. * */ - static class TopoSort implements Comparator { + static class TopoSort implements Comparator, + Serializable { public int compare(LoggedNetworkTopology t1, LoggedNetworkTopology t2) { return t1.name.getValue().compareTo(t2.name.getValue()); } diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TraceBuilder.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TraceBuilder.java index 9a35e84284..2fb5293123 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TraceBuilder.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TraceBuilder.java @@ -20,6 +20,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -98,7 +99,7 @@ static class MyOptions { * history file names should result in the order of jobs' submission times. */ private static class HistoryLogsComparator - implements Comparator { + implements Comparator, Serializable { @Override public int compare(FileStatus file1, FileStatus file2) { return file1.getPath().getName().compareTo( diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java index de3fa9c376..5856626818 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java @@ -27,7 +27,7 @@ * //TODO There is no caching for saving memory. */ public class WordListAnonymizerUtility { - public static final String[] KNOWN_WORDS = + static final String[] KNOWN_WORDS = new String[] {"job", "tmp", "temp", "home", "homes", "usr", "user", "test"}; /** diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/NodeName.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/NodeName.java index 862ed5b829..c0b8d45cf7 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/NodeName.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/NodeName.java @@ -93,16 +93,8 @@ public NodeName(String nodeName) { } public NodeName(String rName, String hName) { - rName = (rName == null) - ? rName - : rName.length() == 0 - ? null - : rName; - hName = (hName == null) - ? hName - : hName.length() == 0 - ? null - : hName; + rName = (rName == null || rName.length() == 0) ? null : rName; + hName = (hName == null || hName.length() == 0) ? null : hName; if (hName == null) { nodeName = rName; rackName = rName; diff --git a/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml new file mode 100644 index 0000000000..122c75bbb1 --- /dev/null +++ b/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml index c19872e630..0ab9f6ccfb 100644 --- a/hadoop-tools/hadoop-streaming/pom.xml +++ b/hadoop-tools/hadoop-streaming/pom.xml @@ -96,6 +96,16 @@ + + org.codehaus.mojo + findbugs-maven-plugin + + true + true + ${basedir}/dev-support/findbugs-exclude.xml + Max + + org.apache.maven.plugins maven-antrun-plugin diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java index 39ba36d179..b014cde622 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java @@ -91,7 +91,7 @@ public class StreamJob implements Tool { @Deprecated public StreamJob(String[] argv, boolean mayExit) { this(); - argv_ = argv; + argv_ = Arrays.copyOf(argv, argv.length); this.config_ = new Configuration(); } @@ -113,7 +113,7 @@ public void setConf(Configuration conf) { @Override public int run(String[] args) throws Exception { try { - this.argv_ = args; + this.argv_ = Arrays.copyOf(args, args.length); init(); preProcessArgs(); @@ -290,7 +290,7 @@ void parseArgv() { LOG.warn("-file option is deprecated, please use generic option" + " -files instead."); - String fileList = null; + StringBuffer fileList = new StringBuffer(); for (String file : values) { packageFiles_.add(file); try { @@ -298,13 +298,15 @@ void parseArgv() { Path path = new Path(pathURI); FileSystem localFs = FileSystem.getLocal(config_); String finalPath = path.makeQualified(localFs).toString(); - fileList = fileList == null ? finalPath : fileList + "," + finalPath; + if(fileList.length() > 0) { + fileList.append(','); + } + fileList.append(finalPath); } catch (Exception e) { throw new IllegalArgumentException(e); } } - config_.set("tmpfiles", config_.get("tmpfiles", "") + - (fileList == null ? "" : fileList)); + config_.set("tmpfiles", config_.get("tmpfiles", "") + fileList); validate(packageFiles_); }