big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-11465. Fix findbugs warnings in hadoop-gridmix. (Contributed by Varun Saxena)

Browse Source
This commit is contained in:
Arpit Agarwal 2015-01-12 21:22:58 -08:00
parent 51881535e6
commit c4cba6165a
8 changed files with 68 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -475,6 +475,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-9992. Modify the NN loadGenerator to optionally run as a MapReduce job
(Akshay Radia via brandonli)
HADOOP-11465. Fix findbugs warnings in hadoop-gridmix. (Varun Saxena via
Arpit Agarwal)
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.

7
hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
View File

@ -30,4 +30,11 @@
<Bug pattern="REC_CATCH_EXCEPTION"/>
<Bug code="REC"/>
</Match>
<!-- This has been done knowingly and meant to fool JVM so that it doesn't optimize code -->
<Match>
<Class name="org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin$DefaultCpuUsageEmulator"/>
<Field name="returnValue"/>
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/>
</Match>
</FindBugsFilter>

6
hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
View File

@ -21,6 +21,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
@ -98,6 +99,8 @@ class CompressionEmulationUtil {
private static final CompressionRatioLookupTable COMPRESSION_LOOKUP_TABLE =
new CompressionRatioLookupTable();
private static final Charset charsetUTF8 = Charset.forName("UTF-8");
/**
* This is a {@link Mapper} implementation for generating random text data.
* It uses {@link RandomTextDataGenerator} for generating text data and the
@ -133,7 +136,8 @@ public void map(NullWritable key, LongWritable value, Context context)
String randomKey = rtg.getRandomWord();
String randomValue = rtg.getRandomWord();
context.write(new Text(randomKey), new Text(randomValue));
bytes -= (randomValue.getBytes().length + randomKey.getBytes().length);
bytes -= (randomValue.getBytes(charsetUTF8).length +
randomKey.getBytes(charsetUTF8).length);
}
}
}

10
hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
View File

@ -41,6 +41,7 @@
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@ -112,6 +113,8 @@ class DistributedCacheEmulator {
Configuration conf; // gridmix configuration
private static final Charset charsetUTF8 = Charset.forName("UTF-8");
// Pseudo local file system where local FS based distributed cache files are
// created by gridmix.
FileSystem pseudoLocalFs = null;
@ -436,9 +439,10 @@ public int compare(Object dc1, Object dc2) {
for (Iterator it = dcFiles.iterator(); it.hasNext();) {
Map.Entry entry = (Map.Entry)it.next();
LongWritable fileSize =
new LongWritable(Long.valueOf(entry.getValue().toString()));
new LongWritable(Long.parseLong(entry.getValue().toString()));
BytesWritable filePath =
new BytesWritable(entry.getKey().toString().getBytes());
new BytesWritable(
entry.getKey().toString().getBytes(charsetUTF8));
byteCount += fileSize.get();
bytesSync += fileSize.get();
@ -515,7 +519,7 @@ void configureDistCacheFiles(Configuration conf, JobConf jobConf)
// local FS based distributed cache file.
// Create this file on the pseudo local FS.
String fileId = MD5Hash.digest(files[i] + timeStamps[i]).toString();
long fileSize = Long.valueOf(fileSizes[i]);
long fileSize = Long.parseLong(fileSizes[i]);
Path mappedLocalFilePath =
PseudoLocalFs.generateFilePath(fileId, fileSize)
.makeQualified(pseudoLocalFs.getUri(),

6
hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.mapred.gridmix;
import java.io.IOException;
import java.nio.charset.Charset;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
@ -95,6 +96,8 @@ class GenerateDistCacheData extends GridmixJob {
*/
static final short GRIDMIX_DISTCACHE_FILE_PERM = 0644;
private static final Charset charsetUTF8 = Charset.forName("UTF-8");
public GenerateDistCacheData(Configuration conf) throws IOException {
super(conf, 0L, JOB_NAME);
}
@ -152,7 +155,8 @@ protected void setup(Context context)
public void map(LongWritable key, BytesWritable value, Context context)
throws IOException, InterruptedException {
String fileName = new String(value.getBytes(), 0, value.getLength());
String fileName = new String(value.getBytes(), 0,
value.getLength(), charsetUTF8);
Path path = new Path(fileName);
FSDataOutputStream dos =

36
hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
View File

@ -729,40 +729,40 @@ protected void printUsage(PrintStream out) {
out.println(" -users <usersResourceURI> : URI that contains the users list.");
out.println("Configuration parameters:");
out.println(" General parameters:");
out.printf(" %-48s : Output directory\n", GRIDMIX_OUT_DIR);
out.printf(" %-48s : Submitting threads\n", GRIDMIX_SUB_THR);
out.printf(" %-48s : Queued job desc\n", GRIDMIX_QUE_DEP);
out.printf(" %-48s : User resolution class\n", GRIDMIX_USR_RSV);
out.printf(" %-48s : Job types (%s)\n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes());
out.printf(" %-48s : Output directory%n", GRIDMIX_OUT_DIR);
out.printf(" %-48s : Submitting threads%n", GRIDMIX_SUB_THR);
out.printf(" %-48s : Queued job desc%n", GRIDMIX_QUE_DEP);
out.printf(" %-48s : User resolution class%n", GRIDMIX_USR_RSV);
out.printf(" %-48s : Job types (%s)%n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes());
out.println(" Parameters related to job submission:");
out.printf(" %-48s : Default queue\n",
out.printf(" %-48s : Default queue%n",
GridmixJob.GRIDMIX_DEFAULT_QUEUE);
out.printf(" %-48s : Enable/disable using queues in trace\n",
out.printf(" %-48s : Enable/disable using queues in trace%n",
GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE);
out.printf(" %-48s : Job submission policy (%s)\n",
out.printf(" %-48s : Job submission policy (%s)%n",
GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, getSubmissionPolicies());
out.println(" Parameters specific for LOADJOB:");
out.printf(" %-48s : Key fraction of rec\n",
out.printf(" %-48s : Key fraction of rec%n",
AvgRecordFactory.GRIDMIX_KEY_FRC);
out.println(" Parameters specific for SLEEPJOB:");
out.printf(" %-48s : Whether to ignore reduce tasks\n",
out.printf(" %-48s : Whether to ignore reduce tasks%n",
SleepJob.SLEEPJOB_MAPTASK_ONLY);
out.printf(" %-48s : Number of fake locations for map tasks\n",
out.printf(" %-48s : Number of fake locations for map tasks%n",
JobCreator.SLEEPJOB_RANDOM_LOCATIONS);
out.printf(" %-48s : Maximum map task runtime in mili-sec\n",
out.printf(" %-48s : Maximum map task runtime in mili-sec%n",
SleepJob.GRIDMIX_SLEEP_MAX_MAP_TIME);
out.printf(" %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)\n",
out.printf(" %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)%n",
SleepJob.GRIDMIX_SLEEP_MAX_REDUCE_TIME);
out.println(" Parameters specific for STRESS submission throttling policy:");
out.printf(" %-48s : jobs vs task-tracker ratio\n",
out.printf(" %-48s : jobs vs task-tracker ratio%n",
StressJobFactory.CONF_MAX_JOB_TRACKER_RATIO);
out.printf(" %-48s : maps vs map-slot ratio\n",
out.printf(" %-48s : maps vs map-slot ratio%n",
StressJobFactory.CONF_OVERLOAD_MAPTASK_MAPSLOT_RATIO);
out.printf(" %-48s : reduces vs reduce-slot ratio\n",
out.printf(" %-48s : reduces vs reduce-slot ratio%n",
StressJobFactory.CONF_OVERLOAD_REDUCETASK_REDUCESLOT_RATIO);
out.printf(" %-48s : map-slot share per job\n",
out.printf(" %-48s : map-slot share per job%n",
StressJobFactory.CONF_MAX_MAPSLOT_SHARE_PER_JOB);
out.printf(" %-48s : reduce-slot share per job\n",
out.printf(" %-48s : reduce-slot share per job%n",
StressJobFactory.CONF_MAX_REDUCESLOT_SHARE_PER_JOB);
}

2
hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
View File

@ -124,7 +124,7 @@ long validateFileNameFormat(Path path) throws FileNotFoundException {
} else {
String[] parts = path.toUri().getPath().split("\\.");
try {
fileSize = Long.valueOf(parts[parts.length - 1]);
fileSize = Long.parseLong(parts[parts.length - 1]);
valid = (fileSize >= 0);
} catch (NumberFormatException e) {
valid = false;

24
hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
View File

@ -120,6 +120,22 @@ public static JobStats generateJobStats(Job job, JobStory jobdesc) {
return new JobStats(maps, reds, job);
}
private static void addToNumMapsSubmitted(int numMaps) {
numMapsSubmitted += numMaps;
}
private static void addToNumReducesSubmitted(int numReduces) {
numReducesSubmitted += numReduces;
}
private static void subtractFromNumMapsSubmitted(int numMaps) {
numMapsSubmitted -= numMaps;
}
private static void subtractFromNumReducesSubmitted(int numReduces) {
numReducesSubmitted -= numReduces;
}
/**
* Add a submitted job for monitoring.
*/
@ -131,8 +147,8 @@ public void addJobStats(JobStats stats) {
return;
}
submittedJobsMap.put(seq, stats);
numMapsSubmitted += stats.getNoOfMaps();
numReducesSubmitted += stats.getNoOfReds();
addToNumMapsSubmitted(stats.getNoOfMaps());
addToNumReducesSubmitted(stats.getNoOfReds());
}
/**
@ -156,8 +172,8 @@ public void add(Statistics.JobStats job) {
}
// update the total number of submitted map/reduce task count
numMapsSubmitted -= stat.getNoOfMaps();
numReducesSubmitted -= stat.getNoOfReds();
subtractFromNumMapsSubmitted(stat.getNoOfMaps());
subtractFromNumReducesSubmitted(stat.getNoOfReds());
completedJobsInCurrentInterval++;
//check if we have reached the maximum level of job completions.