HDDS-1152. Add trace information for the client side of the datanode writes

Contributed by Elek, Marton.

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java

@@ -37,6 +37,9 @@
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Time;
+
+import io.opentracing.Scope;
+import io.opentracing.util.GlobalTracer;
 import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
 import org.apache.ratis.thirdparty.io.grpc.Status;
 import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts;
@@ -288,17 +291,20 @@ private XceiverClientReply sendCommandWithRetry(
   public XceiverClientReply sendCommandAsync(
       ContainerCommandRequestProto request)
       throws IOException, ExecutionException, InterruptedException {
-    XceiverClientReply asyncReply =
+    try (Scope scope = GlobalTracer.get()
-        sendCommandAsync(request, pipeline.getFirstNode());
+        .buildSpan("XceiverClientGrpc." + request.getCmdType().name())
-
+        .startActive(true)) {
-    // TODO : for now make this API sync in nature as async requests are
+      XceiverClientReply asyncReply =
-    // served out of order over XceiverClientGrpc. This needs to be fixed
+          sendCommandAsync(request, pipeline.getFirstNode());
-    // if this API is to be used for I/O path. Currently, this is not
+      // TODO : for now make this API sync in nature as async requests are
-    // used for Read/Write Operation but for tests.
+      // served out of order over XceiverClientGrpc. This needs to be fixed
-    if (!HddsUtils.isReadOnly(request)) {
+      // if this API is to be used for I/O path. Currently, this is not
-      asyncReply.getResponse().get();
+      // used for Read/Write Operation but for tests.
+      if (!HddsUtils.isReadOnly(request)) {
+        asyncReply.getResponse().get();
+      }
+      return asyncReply;
     }
-    return asyncReply;
   }
 
   private XceiverClientReply sendCommandAsync(

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java

@@ -22,6 +22,9 @@
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+
+import io.opentracing.Scope;
+import io.opentracing.util.GlobalTracer;
 import org.apache.ratis.grpc.GrpcTlsConfig;
 import org.apache.ratis.proto.RaftProtos;
 import org.apache.ratis.protocol.RaftRetryFailureException;
@@ -191,15 +194,20 @@ private RaftClient getClient() {
 
   private CompletableFuture<RaftClientReply> sendRequestAsync(
       ContainerCommandRequestProto request) {
-    ContainerCommandRequestProto finalPayload =
+    try (Scope scope = GlobalTracer.get()
-        ContainerCommandRequestProto.newBuilder(request)
+        .buildSpan("XceiverClientRatis." + request.getCmdType().name())
-            .setTraceID(TracingUtil.exportCurrentSpan())
+        .startActive(true)) {
-            .build();
+      ContainerCommandRequestProto finalPayload =
-    boolean isReadOnlyRequest = HddsUtils.isReadOnly(finalPayload);
+          ContainerCommandRequestProto.newBuilder(request)
-    ByteString byteString = finalPayload.toByteString();
+              .setTraceID(TracingUtil.exportCurrentSpan())
-    LOG.debug("sendCommandAsync {} {}", isReadOnlyRequest, finalPayload);
+              .build();
-    return isReadOnlyRequest ? getClient().sendReadOnlyAsync(() -> byteString) :
+      boolean isReadOnlyRequest = HddsUtils.isReadOnly(finalPayload);
-        getClient().sendAsync(() -> byteString);
+      ByteString byteString = finalPayload.toByteString();
+      LOG.debug("sendCommandAsync {} {}", isReadOnlyRequest, finalPayload);
+      return isReadOnlyRequest ?
+          getClient().sendReadOnlyAsync(() -> byteString) :
+          getClient().sendAsync(() -> byteString);
+    }
   }
 
   // gets the minimum log index replicated to all servers

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java

@@ -1,3 +1,4 @@
+
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with this
@@ -614,9 +615,13 @@ public void run() {
                     .update(keyCreationDuration);
                 keyCreationTime.getAndAdd(keyCreationDuration);
                 long keyWriteStart = System.nanoTime();
-                os.write(keyValue);
+                try (Scope writeScope = GlobalTracer.get()
-                os.write(randomValue);
+                    .buildSpan("writeKeyData")
-                os.close();
+                    .startActive(true)) {
+                  os.write(keyValue);
+                  os.write(randomValue);
+                  os.close();
+                }
 
                 long keyWriteDuration = System.nanoTime() - keyWriteStart;