From a1140959dab3f35accbd6c66abfa14f94ff7dcec Mon Sep 17 00:00:00 2001 From: Colin Patrick Mccabe Date: Thu, 28 May 2015 12:00:55 -0700 Subject: [PATCH] HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating (Masatake Iwasaki via Colin P. McCabe) --- .../hadoop-common/CHANGES.txt | 3 ++ .../hadoop/tracing/SpanReceiverHost.java | 45 ++----------------- .../src/site/markdown/Tracing.md | 12 ++--- .../org/apache/hadoop/hdfs/DFSClient.java | 16 ++----- hadoop-project/pom.xml | 2 +- 5 files changed, 18 insertions(+), 60 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 51eff78e76..2b85c1ca66 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -620,6 +620,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12030. test-patch should only report on newly introduced findbugs warnings. (Sean Busbey via aw) + HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating + (Masatake Iwasaki via Colin P. McCabe) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java index bf9479b43a..9a99e05d4a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java @@ -17,33 +17,23 @@ */ package org.apache.hadoop.tracing; -import java.io.BufferedReader; -import java.io.DataInputStream; -import java.io.EOFException; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; -import java.util.UUID; -import org.apache.commons.io.Charsets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.tracing.SpanReceiverInfo.ConfigurationPair; -import org.apache.hadoop.tracing.TraceUtils; -import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.htrace.SpanReceiver; import org.apache.htrace.SpanReceiverBuilder; import org.apache.htrace.Trace; +import org.apache.htrace.impl.LocalFileSpanReceiver; /** * This class provides functions for reading the names of SpanReceivers from @@ -54,7 +44,7 @@ @InterfaceAudience.Private public class SpanReceiverHost implements TraceAdminProtocol { public static final String SPAN_RECEIVERS_CONF_SUFFIX = - "spanreceiver.classes"; + "spanreceiver.classes"; private static final Log LOG = LogFactory.getLog(SpanReceiverHost.class); private static final HashMap hosts = new HashMap(1); @@ -88,33 +78,6 @@ public void run() { private static List EMPTY = Collections.emptyList(); - private static String getUniqueLocalTraceFileName() { - String tmp = System.getProperty("java.io.tmpdir", "/tmp"); - String nonce = null; - BufferedReader reader = null; - try { - // On Linux we can get a unique local file name by reading the process id - // out of /proc/self/stat. (There isn't any portable way to get the - // process ID from Java.) - reader = new BufferedReader( - new InputStreamReader(new FileInputStream("/proc/self/stat"), - Charsets.UTF_8)); - String line = reader.readLine(); - if (line == null) { - throw new EOFException(); - } - nonce = line.split(" ")[0]; - } catch (IOException e) { - } finally { - IOUtils.cleanup(LOG, reader); - } - if (nonce == null) { - // If we can't use the process ID, use a random nonce. - nonce = UUID.randomUUID().toString(); - } - return new File(tmp, nonce).getAbsolutePath(); - } - private SpanReceiverHost(String confPrefix) { this.confPrefix = confPrefix; } @@ -143,7 +106,7 @@ public synchronized void loadSpanReceivers(Configuration conf) { // testing. String pathKey = confPrefix + LOCAL_FILE_SPAN_RECEIVER_PATH_SUFFIX; if (config.get(pathKey) == null) { - String uniqueFile = getUniqueLocalTraceFileName(); + String uniqueFile = LocalFileSpanReceiver.getUniqueLocalTraceFileName(); config.set(pathKey, uniqueFile); if (LOG.isTraceEnabled()) { LOG.trace("Set " + pathKey + " to " + uniqueFile); @@ -191,7 +154,7 @@ public synchronized void closeReceivers() { public synchronized SpanReceiverInfo[] listSpanReceivers() throws IOException { - SpanReceiverInfo info[] = new SpanReceiverInfo[receivers.size()]; + SpanReceiverInfo[] info = new SpanReceiverInfo[receivers.size()]; int i = 0; for(Map.Entry entry : receivers.entrySet()) { diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md b/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md index 76b5ee0b58..bf5685c6c2 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md @@ -78,6 +78,10 @@ You also need to add the jar bundling SpanReceiver to the classpath of Hadoop on each node. (LocalFileSpanReceiver in the example above is included in the jar of htrace-core which is bundled with Hadoop.) +``` + $ cp htrace-htraced/target/htrace-htraced-3.2.0-incubating.jar $HADOOP_HOME/share/hadoop/common/lib/ +``` + ### Dynamic update of tracing configuration You can use `hadoop trace` command to see and update the tracing configuration of each servers. @@ -149,6 +153,7 @@ which start tracing span before invoking HDFS shell command. import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.hdfs.DFSConfigKeys; + import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.tracing.SpanReceiverHost; import org.apache.hadoop.util.ToolRunner; import org.apache.htrace.Sampler; @@ -157,19 +162,16 @@ which start tracing span before invoking HDFS shell command. public class TracingFsShell { public static void main(String argv[]) throws Exception { - Configuration conf = new Configuration(); + Configuration conf = new HdfsConfiguration(); FsShell shell = new FsShell(); conf.setQuietMode(false); shell.setConf(conf); SpanReceiverHost.get(conf, DFSConfigKeys.DFS_SERVER_HTRACE_PREFIX); int res = 0; - TraceScope ts = null; - try { - ts = Trace.startSpan("FsShell", Sampler.ALWAYS); + try (TraceScope ts = Trace.startSpan("FsShell", Sampler.ALWAYS)) { res = ToolRunner.run(shell, argv); } finally { shell.close(); - if (ts != null) ts.close(); } System.exit(res); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 60e5577028..fc1cd26e21 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -37,7 +37,6 @@ import java.net.SocketAddress; import java.net.URI; import java.net.UnknownHostException; -import java.nio.charset.Charset; import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.Collections; @@ -3223,35 +3222,26 @@ public SaslDataTransferClient getSaslDataTransferClient() { return saslClient; } - private static final byte[] PATH = "path".getBytes(Charset.forName("UTF-8")); - TraceScope getPathTraceScope(String description, String path) { TraceScope scope = Trace.startSpan(description, traceSampler); Span span = scope.getSpan(); if (span != null) { if (path != null) { - span.addKVAnnotation(PATH, - path.getBytes(Charset.forName("UTF-8"))); + span.addKVAnnotation("path", path); } } return scope; } - private static final byte[] SRC = "src".getBytes(Charset.forName("UTF-8")); - - private static final byte[] DST = "dst".getBytes(Charset.forName("UTF-8")); - TraceScope getSrcDstTraceScope(String description, String src, String dst) { TraceScope scope = Trace.startSpan(description, traceSampler); Span span = scope.getSpan(); if (span != null) { if (src != null) { - span.addKVAnnotation(SRC, - src.getBytes(Charset.forName("UTF-8"))); + span.addKVAnnotation("src", src); } if (dst != null) { - span.addKVAnnotation(DST, - dst.getBytes(Charset.forName("UTF-8"))); + span.addKVAnnotation("dst", dst); } } return scope; diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 1d0262fcdc..a3470a0e91 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -798,7 +798,7 @@ org.apache.htrace htrace-core - 3.1.0-incubating + 3.2.0-incubating org.jdom