diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 7ce0647b12..574c606f53 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -166,6 +166,8 @@ Release 2.0.3-alpha - Unreleased HADOOP-8911. CRLF characters in source and text files. (Raja Aluri via suresh) + MAPREDUCE-4723. Fix warnings found by findbugs 2. (Sandy Ryza via eli) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml index 73d54038a5..f548885ff4 100644 --- a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml +++ b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml @@ -479,4 +479,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java index e1163a171d..1f9686a016 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java @@ -359,9 +359,8 @@ private void runSubtask(org.apache.hadoop.mapred.Task task, + StringUtils.stringifyException(e)); } // Report back any failures, for diagnostic purposes - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - exception.printStackTrace(new PrintStream(baos)); - umbilical.reportDiagnosticInfo(classicAttemptID, baos.toString()); + umbilical.reportDiagnosticInfo(classicAttemptID, + StringUtils.stringifyException(exception)); throw new RuntimeException(); } catch (Throwable throwable) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java index b4ac5b532b..ab084cfcbd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java @@ -315,8 +315,6 @@ public boolean statusUpdate(TaskAttemptID taskAttemptID, + taskStatus.getProgress()); // Task sends the updated state-string to the TT. taskAttemptStatus.stateString = taskStatus.getStateString(); - // Set the output-size when map-task finishes. Set by the task itself. - taskAttemptStatus.outputSize = taskStatus.getOutputSize(); // Task sends the updated phase to the TT. taskAttemptStatus.phase = TypeConverter.toYarn(taskStatus.getPhase()); // Counters are updated by the task. Convert counters into new format as diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java index 6413bb1a2f..84676f26e7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java @@ -184,10 +184,8 @@ public Object run() throws Exception { LOG.info("Exception cleaning up: " + StringUtils.stringifyException(e)); } // Report back any failures, for diagnostic purposes - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - exception.printStackTrace(new PrintStream(baos)); if (taskid != null) { - umbilical.fatalError(taskid, baos.toString()); + umbilical.fatalError(taskid, StringUtils.stringifyException(exception)); } } catch (Throwable throwable) { LOG.fatal("Error running child : " diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index 7ba3ccd03a..f7860922d7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -600,6 +600,8 @@ public void processEventForJobSummary(HistoryEvent event, JobSummary summary, summary.setJobFinishTime(juce.getFinishTime()); setSummarySlotSeconds(summary, context.getJob(jobId).getAllCounters()); break; + default: + throw new YarnException("Invalid event type"); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java index ac07e94813..715f63d1be 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java @@ -49,7 +49,6 @@ public static class TaskAttemptStatus { public Counters counters; public String stateString; public Phase phase; - public long outputSize; public List fetchFailedMaps; public long mapFinishTime; public long shuffleFinishTime; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index b46ee466ae..ad30c1e95f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -833,6 +833,9 @@ JobStateInternal finished(JobStateInternal finalState) { break; case SUCCEEDED: metrics.completedJob(this); + break; + default: + throw new IllegalArgumentException("Illegal job state: " + finalState); } return finalState; } @@ -1311,6 +1314,9 @@ public void constructFinalFullcounters() { case REDUCE: this.finalReduceCounters.incrAllCounters(counters); break; + default: + throw new IllegalStateException("Task type neither map nor reduce: " + + t.getType()); } this.fullCounters.incrAllCounters(counters); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 7eacc8c827..b400259fd6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -1335,6 +1335,8 @@ public void transition(TaskAttemptImpl taskAttempt, taskAttempt.attemptId, TaskEventType.T_ATTEMPT_KILLED)); break; + default: + LOG.error("Task final state is not FAILED or KILLED: " + finalState); } if (taskAttempt.getLaunchTime() != 0) { TaskAttemptUnsuccessfulCompletionEvent tauce = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java index ea3101a68d..59c9795e2d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java @@ -210,7 +210,7 @@ protected void containerFailedOnHost(String hostName) { return; //already blacklisted } Integer failures = nodeFailures.remove(hostName); - failures = failures == null ? 0 : failures; + failures = failures == null ? Integer.valueOf(0) : failures; failures++; LOG.info(failures + " failures on node " + hostName); if (failures >= maxTaskFailuresPerNode) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java index a71604724f..aee4821996 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java @@ -43,7 +43,6 @@ abstract class StartEndTimesBase implements TaskRuntimeEstimator { static final int MINIMUM_COMPLETE_NUMBER_TO_SPECULATE = 1; - protected Configuration conf = null; protected AppContext context = null; protected final Map startTimes @@ -82,7 +81,6 @@ public long attemptEnrolledTime(TaskAttemptId attemptID) { @Override public void contextualize(Configuration conf, AppContext context) { - this.conf = conf; this.context = context; Map allJobs = context.getAllJobs(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java index a92902009a..50ebd16aa6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java @@ -285,6 +285,8 @@ private void countTasksAndAttempts(Job job) { case SCHEDULED: ++this.mapsPending; break; + default: + break; } break; case REDUCE: @@ -296,8 +298,13 @@ private void countTasksAndAttempts(Job job) { case SCHEDULED: ++this.reducesPending; break; + default: + break; } break; + default: + throw new IllegalStateException( + "Task type is neither map nor reduce: " + task.getType()); } // Attempts counts Map attempts = task.getAttempts(); @@ -337,6 +344,9 @@ private void countTasksAndAttempts(Job job) { this.failedReduceAttempts += failed; this.killedReduceAttempts += killed; break; + default: + throw new IllegalStateException("Task type neither map nor reduce: " + + task.getType()); } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java index 4b1a1a0bd9..0c9832477a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java @@ -412,7 +412,6 @@ private void updateStatus(MRApp app, TaskAttempt attempt, Phase phase) { status.fetchFailedMaps = new ArrayList(); status.id = attempt.getID(); status.mapFinishTime = 0; - status.outputSize = 0; status.phase = phase; status.progress = 0.5f; status.shuffleFinishTime = 0; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java index 27fcec2c6a..34b8dc7635 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java @@ -86,7 +86,6 @@ public void test() throws Exception { taskAttemptStatus.stateString = "RUNNING"; taskAttemptStatus.taskState = TaskAttemptState.RUNNING; taskAttemptStatus.phase = Phase.MAP; - taskAttemptStatus.outputSize = 3; // send the status update app.getContext().getEventHandler().handle( new TaskAttemptStatusUpdateEvent(attempt.getID(), taskAttemptStatus)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 596802853f..7d863a56e4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -59,6 +59,8 @@ import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; +import com.google.common.base.Charsets; + /** * Helper class for MR applications */ @@ -159,7 +161,8 @@ private static void setMRFrameworkClasspath( } if (classpathFileStream != null) { - reader = new BufferedReader(new InputStreamReader(classpathFileStream)); + reader = new BufferedReader(new InputStreamReader(classpathFileStream, + Charsets.UTF_8)); String cp = reader.readLine(); if (cp != null) { Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java index 4f7e4b95fd..d06d6fb28b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java @@ -420,6 +420,8 @@ private static String[] getPathStrings(String commaSeparatedPaths) { } break; } + default: + continue; // nothing special to do for this character } } pathStrings.add(commaSeparatedPaths.substring(pathStart, length)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java index 115cf3e05c..097e338a92 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java @@ -18,6 +18,7 @@ package org.apache.hadoop.mapred; import java.io.IOException; +import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.Writer; import java.util.List; @@ -30,6 +31,8 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import com.google.common.base.Charsets; + /** * JobQueueClient is interface provided to the user in order to get * JobQueue related information from the {@link JobTracker} @@ -144,7 +147,8 @@ void printJobQueueInfo(JobQueueInfo jobQueueInfo, Writer writer, private void displayQueueList() throws IOException { JobQueueInfo[] rootQueues = jc.getRootQueues(); for (JobQueueInfo queue : rootQueues) { - printJobQueueInfo(queue, new PrintWriter(System.out)); + printJobQueueInfo(queue, new PrintWriter(new OutputStreamWriter( + System.out, Charsets.UTF_8))); } } @@ -182,7 +186,8 @@ private void displayQueueInfo(String queue, boolean showJobs) System.out.println("Queue \"" + queue + "\" does not exist."); return; } - printJobQueueInfo(jobQueueInfo, new PrintWriter(System.out)); + printJobQueueInfo(jobQueueInfo, new PrintWriter(new OutputStreamWriter( + System.out, Charsets.UTF_8))); if (showJobs && (jobQueueInfo.getChildren() == null || jobQueueInfo.getChildren().size() == 0)) { JobStatus[] jobs = jobQueueInfo.getJobStatuses(); @@ -223,10 +228,10 @@ private void displayUsage(String cmd) { if ("-queueinfo".equals(cmd)) { System.err.println(prefix + "[" + cmd + " [-showJobs]]"); } else { - System.err.printf(prefix + " \n"); - System.err.printf("\t[-list]\n"); - System.err.printf("\t[-info [-showJobs]]\n"); - System.err.printf("\t[-showacls] \n\n"); + System.err.printf(prefix + " %n"); + System.err.printf("\t[-list]%n"); + System.err.printf("\t[-info [-showJobs]]%n"); + System.err.printf("\t[-showacls] %n%n"); ToolRunner.printGenericCommandUsage(System.out); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java index 12ad047999..12d686733f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java @@ -49,6 +49,8 @@ import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import com.google.common.base.Charsets; + /** * A simple logger to handle the task-specific user logs. * This class uses the system property hadoop.log.dir. @@ -104,7 +106,8 @@ private static LogFileDetail getLogFileDetail(TaskAttemptID taskid, throws IOException { File indexFile = getIndexFile(taskid, isCleanup); BufferedReader fis = new BufferedReader(new InputStreamReader( - SecureIOUtils.openForRead(indexFile, obtainLogDirOwner(taskid), null))); + SecureIOUtils.openForRead(indexFile, obtainLogDirOwner(taskid), null), + Charsets.UTF_8)); //the format of the index file is //LOG_DIR: //stdout: diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TextInputFormat.java index 02cffb84ad..5c871f0f46 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TextInputFormat.java @@ -27,6 +27,8 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.*; +import com.google.common.base.Charsets; + /** * An {@link InputFormat} for plain text files. Files are broken into lines. * Either linefeed or carriage-return are used to signal end of line. Keys are @@ -59,7 +61,9 @@ public RecordReader getRecordReader( reporter.setStatus(genericSplit.toString()); String delimiter = job.get("textinputformat.record.delimiter"); byte[] recordDelimiterBytes = null; - if (null != delimiter) recordDelimiterBytes = delimiter.getBytes(); + if (null != delimiter) { + recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8); + } return new LineRecordReader(job, (FileSplit) genericSplit, recordDelimiterBytes); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java index 1abaef260c..37980dddd9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java @@ -49,9 +49,7 @@ public class CombineFileRecordReader implements RecordReader { protected CombineFileSplit split; protected JobConf jc; protected Reporter reporter; - protected Class> rrClass; protected Constructor> rrConstructor; - protected FileSystem fs; protected int idx; protected long progress; @@ -106,7 +104,6 @@ public CombineFileRecordReader(JobConf job, CombineFileSplit split, throws IOException { this.split = split; this.jc = job; - this.rrClass = rrClass; this.reporter = reporter; this.idx = 0; this.curReader = null; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java index 23be73a752..62415f935f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java @@ -56,6 +56,8 @@ import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; +import com.google.common.base.Charsets; + @InterfaceAudience.Private @InterfaceStability.Unstable class JobSubmitter { @@ -550,7 +552,7 @@ private void readTokensFromFiles(Configuration conf, Credentials credentials) for(Map.Entry ent: nm.entrySet()) { credentials.addSecretKey(new Text(ent.getKey()), ent.getValue() - .getBytes()); + .getBytes(Charsets.UTF_8)); } } catch (JsonMappingException e) { json_error = true; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java index 82a948cc6f..92490a59e0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java @@ -188,7 +188,7 @@ private void printCounters(StringBuffer buff, Counters totalCounters, decimal.format(counter.getValue()); buff.append( - String.format("\n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", + String.format("%n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", totalGroup.getDisplayName(), counter.getDisplayName(), mapValue, reduceValue, totalValue)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBInputFormat.java index 1688a149d6..d1b9d76394 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBInputFormat.java @@ -30,6 +30,8 @@ import java.util.ArrayList; import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.InputFormat; @@ -58,6 +60,8 @@ public class DBInputFormat extends InputFormat implements Configurable { + private static final Log LOG = LogFactory.getLog(DBInputFormat.class); + private String dbProductName = "DEFAULT"; /** @@ -354,6 +358,8 @@ protected void closeConnection() { this.connection.close(); this.connection = null; } - } catch (SQLException sqlE) { } // ignore exception on close. + } catch (SQLException sqlE) { + LOG.debug("Exception on close", sqlE); + } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java index 984c9cc2dd..42ab9f3b95 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java @@ -219,7 +219,6 @@ public List getSplits(JobContext job) Path p = fs.makeQualified(paths[i]); newpaths.add(p); } - paths = null; // In one single iteration, process all the paths in a single pool. // Processing one pool at a time ensures that a split contains paths diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java index fb86cbafc1..8749362c22 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java @@ -46,9 +46,7 @@ public class CombineFileRecordReader extends RecordReader { Integer.class}; protected CombineFileSplit split; - protected Class> rrClass; protected Constructor> rrConstructor; - protected FileSystem fs; protected TaskAttemptContext context; protected int idx; @@ -111,7 +109,6 @@ public CombineFileRecordReader(CombineFileSplit split, throws IOException { this.split = split; this.context = context; - this.rrClass = rrClass; this.idx = 0; this.curReader = null; this.progress = 0; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java index d86ad156bd..9c4809a22e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java @@ -425,6 +425,8 @@ private static String[] getPathStrings(String commaSeparatedPaths) { } break; } + default: + continue; // nothing special to do for this character } } pathStrings.add(commaSeparatedPaths.substring(pathStart, length)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TextInputFormat.java index 07840d0991..b55d4e442e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TextInputFormat.java @@ -32,6 +32,8 @@ import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import com.google.common.base.Charsets; + /** An {@link InputFormat} for plain text files. Files are broken into lines. * Either linefeed or carriage-return are used to signal end of line. Keys are * the position in the file, and values are the line of text.. */ @@ -47,7 +49,7 @@ public class TextInputFormat extends FileInputFormat { "textinputformat.record.delimiter"); byte[] recordDelimiterBytes = null; if (null != delimiter) - recordDelimiterBytes = delimiter.getBytes(); + recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8); return new LineRecordReader(recordDelimiterBytes); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java index c4c07dddc8..d971facd97 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java @@ -18,21 +18,23 @@ package org.apache.hadoop.mapreduce.security; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.PrintStream; import java.net.URL; import javax.crypto.SecretKey; import javax.servlet.http.HttpServletRequest; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; import org.apache.hadoop.record.Utils; +import com.google.common.base.Charsets; + /** * * utilities for generating kyes, hashes and verifying them for shuffle @@ -41,6 +43,8 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class SecureShuffleUtils { + private static final Log LOG = LogFactory.getLog(SecureShuffleUtils.class); + public static final String HTTP_HEADER_URL_HASH = "UrlHash"; public static final String HTTP_HEADER_REPLY_URL_HASH = "ReplyHash"; @@ -49,7 +53,8 @@ public class SecureShuffleUtils { * @param msg */ public static String generateHash(byte[] msg, SecretKey key) { - return new String(Base64.encodeBase64(generateByteHash(msg, key))); + return new String(Base64.encodeBase64(generateByteHash(msg, key)), + Charsets.UTF_8); } /** @@ -80,7 +85,7 @@ private static boolean verifyHash(byte[] hash, byte[] msg, SecretKey key) { */ public static String hashFromString(String enc_str, SecretKey key) throws IOException { - return generateHash(enc_str.getBytes(), key); + return generateHash(enc_str.getBytes(Charsets.UTF_8), key); } /** @@ -91,9 +96,9 @@ public static String hashFromString(String enc_str, SecretKey key) */ public static void verifyReply(String base64Hash, String msg, SecretKey key) throws IOException { - byte[] hash = Base64.decodeBase64(base64Hash.getBytes()); + byte[] hash = Base64.decodeBase64(base64Hash.getBytes(Charsets.UTF_8)); - boolean res = verifyHash(hash, msg.getBytes(), key); + boolean res = verifyHash(hash, msg.getBytes(Charsets.UTF_8), key); if(res != true) { throw new IOException("Verification of the hashReply failed"); @@ -126,19 +131,4 @@ public static String buildMsgFrom(HttpServletRequest request ) { private static String buildMsgFrom(String uri_path, String uri_query, int port) { return String.valueOf(port) + uri_path + "?" + uri_query; } - - - /** - * byte array to Hex String - * @param ba - * @return string with HEX value of the key - */ - public static String toHex(byte[] ba) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - PrintStream ps = new PrintStream(baos); - for(byte b: ba) { - ps.printf("%x", b); - } - return baos.toString(); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java index ceada74f79..e582d2856f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java @@ -144,7 +144,6 @@ public RawKeyValueIterator run() throws IOException, InterruptedException { for (Fetcher fetcher : fetchers) { fetcher.shutDown(); } - fetchers = null; // stop the scheduler scheduler.close(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java index d6061c52a1..708214088d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java @@ -18,6 +18,7 @@ package org.apache.hadoop.mapreduce.tools; import java.io.IOException; +import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; @@ -53,6 +54,8 @@ import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.logaggregation.LogDumper; +import com.google.common.base.Charsets; + /** * Interprets the map reduce cli options */ @@ -426,25 +429,25 @@ private void displayUsage(String cmd) { " ]. " + " is optional to get task attempt logs."); } else { - System.err.printf(prefix + " \n"); - System.err.printf("\t[-submit ]\n"); - System.err.printf("\t[-status ]\n"); - System.err.printf("\t[-counter ]\n"); - System.err.printf("\t[-kill ]\n"); + System.err.printf(prefix + " %n"); + System.err.printf("\t[-submit ]%n"); + System.err.printf("\t[-status ]%n"); + System.err.printf("\t[-counter ]%n"); + System.err.printf("\t[-kill ]%n"); System.err.printf("\t[-set-priority ]. " + - "Valid values for priorities are: " + jobPriorityValues + "\n"); - System.err.printf("\t[-events <#-of-events>]\n"); - System.err.printf("\t[-history ]\n"); - System.err.printf("\t[-list [all]]\n"); - System.err.printf("\t[-list-active-trackers]\n"); - System.err.printf("\t[-list-blacklisted-trackers]\n"); + "Valid values for priorities are: " + jobPriorityValues + "%n"); + System.err.printf("\t[-events <#-of-events>]%n"); + System.err.printf("\t[-history ]%n"); + System.err.printf("\t[-list [all]]%n"); + System.err.printf("\t[-list-active-trackers]%n"); + System.err.printf("\t[-list-blacklisted-trackers]%n"); System.err.println("\t[-list-attempt-ids " + "]. " + "Valid values for are " + taskTypes + ". " + "Valid values for are " + taskStates); - System.err.printf("\t[-kill-task ]\n"); - System.err.printf("\t[-fail-task ]\n"); - System.err.printf("\t[-logs ]\n\n"); + System.err.printf("\t[-kill-task ]%n"); + System.err.printf("\t[-fail-task ]%n"); + System.err.printf("\t[-logs ]%n%n"); ToolRunner.printGenericCommandUsage(System.out); } } @@ -584,7 +587,8 @@ protected void displayTasks(Job job, String type, String state) public void displayJobList(JobStatus[] jobs) throws IOException, InterruptedException { - displayJobList(jobs, new PrintWriter(System.out)); + displayJobList(jobs, new PrintWriter(new OutputStreamWriter(System.out, + Charsets.UTF_8))); } @Private diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java index 280b7b679b..7898dcae5c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java @@ -19,9 +19,10 @@ package org.apache.hadoop.mapreduce.util; import java.io.BufferedReader; +import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileReader; import java.io.IOException; +import java.io.InputStreamReader; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -30,6 +31,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import com.google.common.base.Charsets; + /** * Plugin to calculate resource information on Linux systems. */ @@ -152,9 +155,10 @@ private void readProcMemInfoFile(boolean readAgain) { // Read "/proc/memInfo" file BufferedReader in = null; - FileReader fReader = null; + InputStreamReader fReader = null; try { - fReader = new FileReader(procfsMemFile); + fReader = new InputStreamReader(new FileInputStream(procfsMemFile), + Charsets.UTF_8); in = new BufferedReader(fReader); } catch (FileNotFoundException f) { // shouldn't happen.... @@ -211,9 +215,10 @@ private void readProcCpuInfoFile() { } // Read "/proc/cpuinfo" file BufferedReader in = null; - FileReader fReader = null; + InputStreamReader fReader = null; try { - fReader = new FileReader(procfsCpuFile); + fReader = new InputStreamReader(new FileInputStream(procfsCpuFile), + Charsets.UTF_8); in = new BufferedReader(fReader); } catch (FileNotFoundException f) { // shouldn't happen.... @@ -258,9 +263,10 @@ private void readProcCpuInfoFile() { private void readProcStatFile() { // Read "/proc/stat" file BufferedReader in = null; - FileReader fReader = null; + InputStreamReader fReader = null; try { - fReader = new FileReader(procfsStatFile); + fReader = new InputStreamReader(new FileInputStream(procfsStatFile), + Charsets.UTF_8); in = new BufferedReader(fReader); } catch (FileNotFoundException f) { // shouldn't happen.... diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java index 076d0b8a9e..2f8b84d5e1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java @@ -26,7 +26,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.util.Shell.ShellCommandExecutor; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java index 583e95d8d1..99c2e7e180 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java @@ -20,9 +20,10 @@ import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileReader; import java.io.IOException; +import java.io.InputStreamReader; import java.math.BigInteger; import java.util.ArrayList; import java.util.List; @@ -39,6 +40,8 @@ import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.util.StringUtils; +import com.google.common.base.Charsets; + /** * A Proc file-system based ProcessTree. Works only on Linux. */ @@ -350,7 +353,7 @@ public void destroy(boolean inBackground) { } private static final String PROCESSTREE_DUMP_FORMAT = - "\t|- %s %s %d %d %s %d %d %d %d %s\n"; + "\t|- %s %s %d %d %s %d %d %d %d %s%n"; /** * Get a dump of the process-tree. @@ -363,7 +366,7 @@ public String getProcessTreeDump() { // The header. ret.append(String.format("\t|- PID PPID PGRPID SESSID CMD_NAME " + "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) " - + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n")); + + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE%n")); for (ProcessInfo p : processTree.values()) { if (p != null) { ret.append(String.format(PROCESSTREE_DUMP_FORMAT, p.getPid(), p @@ -505,10 +508,11 @@ private static ProcessInfo constructProcessInfo(ProcessInfo pinfo, ProcessInfo ret = null; // Read "procfsDir//stat" file - typically /proc//stat BufferedReader in = null; - FileReader fReader = null; + InputStreamReader fReader = null; try { File pidDir = new File(procfsDir, pinfo.getPid()); - fReader = new FileReader(new File(pidDir, PROCFS_STAT_FILE)); + fReader = new InputStreamReader(new FileInputStream( + new File(pidDir, PROCFS_STAT_FILE)), Charsets.UTF_8); in = new BufferedReader(fReader); } catch (FileNotFoundException f) { // The process vanished in the interim! @@ -695,11 +699,11 @@ public String getCmdLine(String procfsDir) { return ret; } BufferedReader in = null; - FileReader fReader = null; + InputStreamReader fReader = null; try { - fReader = - new FileReader(new File(new File(procfsDir, pid), - PROCFS_CMDLINE_FILE)); + fReader = new InputStreamReader(new FileInputStream( + new File(new File(procfsDir, pid), PROCFS_CMDLINE_FILE)), + Charsets.UTF_8); } catch (FileNotFoundException f) { // The process vanished in the interim! return ret; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index f7bc50609a..72cc9eddb6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -535,8 +535,9 @@ private void addDirectoryToSerialNumberIndex(Path serialDirPath) { if (serialPart == null) { LOG.warn("Could not find serial portion from path: " + serialDirPath.toString() + ". Continuing with next"); + } else { + serialNumberIndex.add(serialPart, timestampPart); } - serialNumberIndex.add(serialPart, timestampPart); } private void addDirectoryToJobListCache(Path path) throws IOException { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java index 10ca394439..53dd2ffd4c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java @@ -38,7 +38,7 @@ */ public class HsJobsBlock extends HtmlBlock { final AppContext appContext; - static final SimpleDateFormat dateFormat = + final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss z"); @Inject HsJobsBlock(AppContext appCtx) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index 63da26c17b..61cccd36be 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -106,6 +106,7 @@ import org.jboss.netty.handler.stream.ChunkedWriteHandler; import org.jboss.netty.util.CharsetUtil; +import com.google.common.base.Charsets; import com.google.common.util.concurrent.ThreadFactoryBuilder; public class ShuffleHandler extends AbstractService @@ -490,7 +491,8 @@ private void verifyRequest(String appid, ChannelHandlerContext ctx, SecureShuffleUtils.verifyReply(urlHashStr, enc_str, tokenSecret); // verification passed - encode the reply String reply = - SecureShuffleUtils.generateHash(urlHashStr.getBytes(), tokenSecret); + SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8), + tokenSecret); response.setHeader(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH, reply); if (LOG.isDebugEnabled()) { int len = reply.length(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml index 244adafebd..1215a3ef18 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml @@ -60,4 +60,10 @@ + + + + + + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml index cd9161b742..5b2231e0db 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml @@ -103,6 +103,11 @@ hsqldb provided + + com.google.guava + guava + provided + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java index 0affac0b44..36a953cb74 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java @@ -22,7 +22,9 @@ import java.io.DataOutput; import java.io.IOException; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.io.PrintStream; +import java.io.PrintWriter; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -50,6 +52,8 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import com.google.common.base.Charsets; + /** * A map/reduce program that uses Bailey-Borwein-Plouffe to compute exact * digits of Pi. @@ -151,7 +155,8 @@ protected void cleanup(Context context LOG.info("Writing text output to " + outfile); final OutputStream outputstream = fs.create(outfile); try { - final PrintStream out = new PrintStream(outputstream, true); + final PrintWriter out = new PrintWriter( + new OutputStreamWriter(outputstream, Charsets.UTF_8), true); // write hex text print(out, hex.iterator(), "Pi = 0x3.", "%02X", 5, 5); out.println("Total number of hexadecimal digits is " @@ -184,7 +189,7 @@ public void remove() { } /** Print out elements in a nice format. */ - private static void print(PrintStream out, Iterator iterator, + private static void print(PrintWriter out, Iterator iterator, String prefix, String format, int elementsPerGroup, int groupsPerLine) { final StringBuilder sb = new StringBuilder("\n"); for (int i = 0; i < prefix.length(); i++) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java index b1f7a67a53..fd7e9b692d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java @@ -37,6 +37,8 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import com.google.common.base.Charsets; + public class WordMean extends Configured implements Tool { private double mean = 0; @@ -125,7 +127,7 @@ private double readAndCalcMean(Path path, Configuration conf) // average = total sum / number of elements; try { - br = new BufferedReader(new InputStreamReader(fs.open(file))); + br = new BufferedReader(new InputStreamReader(fs.open(file), Charsets.UTF_8)); long count = 0; long length = 0; @@ -151,7 +153,9 @@ private double readAndCalcMean(Path path, Configuration conf) System.out.println("The mean is: " + theMean); return theMean; } finally { - br.close(); + if (br != null) { + br.close(); + } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java index d22208680f..36a1471999 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java @@ -38,6 +38,8 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import com.google.common.base.Charsets; + public class WordMedian extends Configured implements Tool { private double median = 0; @@ -127,7 +129,7 @@ private double readAndFindMedian(String path, int medianIndex1, BufferedReader br = null; try { - br = new BufferedReader(new InputStreamReader(fs.open(file))); + br = new BufferedReader(new InputStreamReader(fs.open(file), Charsets.UTF_8)); int num = 0; String line; @@ -157,7 +159,9 @@ private double readAndFindMedian(String path, int medianIndex1, } } } finally { - br.close(); + if (br != null) { + br.close(); + } } // error, no median found return -1; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java index d45935ebc0..50a8f1edcc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java @@ -37,6 +37,8 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import com.google.common.base.Charsets; + public class WordStandardDeviation extends Configured implements Tool { private double stddev = 0; @@ -135,7 +137,7 @@ private double readAndCalcStdDev(Path path, Configuration conf) double stddev = 0; BufferedReader br = null; try { - br = new BufferedReader(new InputStreamReader(fs.open(file))); + br = new BufferedReader(new InputStreamReader(fs.open(file), Charsets.UTF_8)); long count = 0; long length = 0; long square = 0; @@ -166,7 +168,9 @@ private double readAndCalcStdDev(Path path, Configuration conf) stddev = Math.sqrt((term - mean)); System.out.println("The standard deviation is: " + stddev); } finally { - br.close(); + if (br != null) { + br.close(); + } } return stddev; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java index 5f76d87ea2..94c647711d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java @@ -33,6 +33,8 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.*; +import com.google.common.base.Charsets; + /** * Launch a distributed pentomino solver. * It generates a complete list of prefixes of length N with each unique prefix @@ -137,9 +139,9 @@ private static long createInputDirectory(FileSystem fs, fs.mkdirs(dir); List splits = pent.getSplits(depth); Path input = new Path(dir, "part1"); - PrintStream file = - new PrintStream(new BufferedOutputStream - (fs.create(input), 64*1024)); + PrintWriter file = + new PrintWriter(new OutputStreamWriter(new BufferedOutputStream + (fs.create(input), 64*1024), Charsets.UTF_8)); for(int[] prefix: splits) { for(int i=0; i < prefix.length; ++i) { if (i != 0) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Sudoku.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Sudoku.java index 692048c0e4..a835b6a6c9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Sudoku.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Sudoku.java @@ -21,6 +21,8 @@ import java.io.*; import java.util.*; +import com.google.common.base.Charsets; + /** * This class uses the dancing links algorithm from Knuth to solve sudoku * puzzles. It has solved 42x42 puzzles in 1.02 seconds. @@ -133,7 +135,8 @@ public void solution(List> names) { * @param stream The input stream to read the data from */ public Sudoku(InputStream stream) throws IOException { - BufferedReader file = new BufferedReader(new InputStreamReader(stream)); + BufferedReader file = new BufferedReader( + new InputStreamReader(stream, Charsets.UTF_8)); String line = file.readLine(); List result = new ArrayList(); while (line != null) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java index f7f907640f..187520a399 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java @@ -19,9 +19,11 @@ import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.util.ArrayList; import java.util.Collections; @@ -32,6 +34,8 @@ import org.apache.hadoop.examples.pi.math.Bellard; import org.apache.hadoop.examples.pi.math.Bellard.Parameter; +import com.google.common.base.Charsets; + /** A class for parsing outputs */ public final class Parser { static final String VERBOSE_PROPERTY = "pi.parser.verbose"; @@ -71,7 +75,8 @@ private void parse(File f, Map> sums) throws IOExcep for(Parameter p : Parameter.values()) m.put(p, new ArrayList()); - final BufferedReader in = new BufferedReader(new FileReader(f)); + final BufferedReader in = new BufferedReader( + new InputStreamReader(new FileInputStream(f), Charsets.UTF_8)); try { for(String line; (line = in.readLine()) != null; ) try { @@ -127,7 +132,8 @@ Map> parse(String inputpath, String outputdir Collections.sort(results); final PrintWriter out = new PrintWriter( - new FileWriter(new File(outputdir, p + ".txt")), true); + new OutputStreamWriter(new FileOutputStream( + new File(outputdir, p + ".txt")), Charsets.UTF_8), true); try { for(int i = 0; i < results.size(); i++) out.println(DistSum.taskResult2string(p + "." + i, results.get(i))); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java index 7963f07e34..8afc1bd760 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java @@ -19,9 +19,10 @@ import java.io.BufferedReader; import java.io.File; -import java.io.FileWriter; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.io.PrintStream; import java.io.PrintWriter; import java.text.SimpleDateFormat; @@ -46,6 +47,8 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.util.ToolRunner; +import com.google.common.base.Charsets; + /** Utility methods */ public class Util { /** Output stream */ @@ -81,7 +84,7 @@ public synchronized long tick(String s) { final long t = System.currentTimeMillis(); final long delta = t - (isAccumulative? start: previous); if (s != null) { - out.format("%15dms (=%-15s: %s\n", delta, millis2String(delta) + ")", s); + out.format("%15dms (=%-15s: %s%n", delta, millis2String(delta) + ")", s); out.flush(); } previous = t; @@ -203,16 +206,16 @@ public static void checkDirectory(File dir) { throw new IllegalArgumentException("dir (=" + dir + ") is not a directory."); } - private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("-yyyyMMdd-HHmmssSSS"); /** Create a writer of a local file. */ public static PrintWriter createWriter(File dir, String prefix) throws IOException { checkDirectory(dir); - + + SimpleDateFormat dateFormat = new SimpleDateFormat("-yyyyMMdd-HHmmssSSS"); for(;;) { final File f = new File(dir, - prefix + DATE_FORMAT.format(new Date(System.currentTimeMillis())) + ".txt"); + prefix + dateFormat.format(new Date(System.currentTimeMillis())) + ".txt"); if (!f.exists()) - return new PrintWriter(new FileWriter(f)); + return new PrintWriter(new OutputStreamWriter(new FileOutputStream(f), Charsets.UTF_8)); try {Thread.sleep(10);} catch (InterruptedException e) {} } @@ -286,7 +289,8 @@ static List readJobOutputs(FileSystem fs, Path outdir) throws IOExce final List results = new ArrayList(); for(FileStatus status : fs.listStatus(outdir)) { if (status.getPath().getName().startsWith("part-")) { - final BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(status.getPath()))); + final BufferedReader in = new BufferedReader( + new InputStreamReader(fs.open(status.getPath()), Charsets.UTF_8)); try { for(String line; (line = in.readLine()) != null; ) results.add(TaskResult.valueOf(line)); @@ -305,7 +309,7 @@ static List readJobOutputs(FileSystem fs, Path outdir) throws IOExce static void writeResults(String name, List results, FileSystem fs, String dir) throws IOException { final Path outfile = new Path(dir, name + ".txt"); Util.out.println(name + "> writing results to " + outfile); - final PrintStream out = new PrintStream(fs.create(outfile), true); + final PrintWriter out = new PrintWriter(new OutputStreamWriter(fs.create(outfile), Charsets.UTF_8), true); try { for(TaskResult r : results) out.println(r); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java index 180c924378..82a451246c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java @@ -29,6 +29,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; +import com.google.common.base.Charsets; + class TeraScheduler { static String USE = "mapreduce.terasort.use.terascheduler"; private static final Log LOG = LogFactory.getLog(TeraScheduler.class); @@ -73,7 +75,8 @@ public String toString() { List readFile(String filename) throws IOException { List result = new ArrayList(10000); - BufferedReader in = new BufferedReader(new FileReader(filename)); + BufferedReader in = new BufferedReader( + new InputStreamReader(new FileInputStream(filename), Charsets.UTF_8)); String line = in.readLine(); while (line != null) { result.add(line);