HDDS-76. Modify SCMStorageReportProto to include the data dir paths as well as the StorageType info. Contributed by Shashikant Banerjee.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java

@@ -108,12 +108,14 @@ public Path getDataPath(String containerName) throws IOException {
 
   @Override
   public StorageLocationReport[] getLocationReport() throws IOException {
+    boolean failed;
     StorageLocationReport[] reports =
         new StorageLocationReport[dataLocations.size()];
     for (int idx = 0; idx < dataLocations.size(); idx++) {
       ContainerStorageLocation loc = dataLocations.get(idx);
       long scmUsed = 0;
       long remaining = 0;
+      failed = false;
       try {
         scmUsed = loc.getScmUsed();
         remaining = loc.getAvailable();
@@ -123,13 +125,19 @@ public StorageLocationReport[] getLocationReport() throws IOException {
         // reset scmUsed and remaining if df/du failed.
         scmUsed = 0;
         remaining = 0;
+        failed = true;
       }
 
-      // TODO: handle failed storage
-      // For now, include storage report for location that failed to get df/du.
-      StorageLocationReport r = new StorageLocationReport(
-          loc.getStorageUuId(), false, loc.getCapacity(),
-          scmUsed, remaining);
+      StorageLocationReport.Builder builder =
+          StorageLocationReport.newBuilder();
+      builder.setStorageLocation(loc.getStorageLocation())
+          .setId(loc.getStorageUuId())
+          .setFailed(failed)
+          .setCapacity(loc.getCapacity())
+          .setRemaining(remaining)
+          .setScmUsed(scmUsed)
+          .setStorageType(loc.getStorageType());
+      StorageLocationReport r = builder.build();
       reports[idx] = r;
     }
     return reports;

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java

@@ -22,6 +22,7 @@
 import com.google.common.base.Preconditions;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.container.common.helpers
     .StorageContainerException;
@@ -38,6 +39,8 @@
     .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMStorageReport;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.StorageTypeProto;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
@@ -818,11 +821,7 @@ public SCMNodeReport getNodeReport() throws IOException {
     SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder();
     for (int i = 0; i < reports.length; i++) {
       SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
-      nrb.addStorageReport(i, srb.setStorageUuid(reports[i].getId())
-          .setCapacity(reports[i].getCapacity())
-          .setScmUsed(reports[i].getScmUsed())
-          .setRemaining(reports[i].getRemaining())
-          .build());
+      nrb.addStorageReport(reports[i].getProtoBufMessage());
     }
     return nrb.build();
   }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java

@@ -22,6 +22,7 @@
 import org.apache.hadoop.fs.CachingGetSpaceUsed;
 import org.apache.hadoop.fs.DF;
 import org.apache.hadoop.fs.GetSpaceUsed;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.io.IOUtils;
@@ -121,6 +122,14 @@ public long getScmUsed() throws IOException{
     return scmUsage.getUsed();
   }
 
+  public String getStorageLocation() {
+    return getNormalizedUri().getRawPath();
+  }
+
+  public StorageType getStorageType() {
+    return dataLocation.getStorageType();
+  }
+
   public void shutdown() {
     saveScmUsed();
     scmUsedSaved = true;

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java

@@ -18,26 +18,38 @@
 
 package org.apache.hadoop.ozone.container.common.impl;
 
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdds.protocol.proto.
+    StorageContainerDatanodeProtocolProtos.SCMStorageReport;
+import org.apache.hadoop.hdds.protocol.proto.
+    StorageContainerDatanodeProtocolProtos.StorageTypeProto;
+
+import java.io.IOException;
+
 /**
  * Storage location stats of datanodes that provide back store for containers.
  *
  */
 public class StorageLocationReport {
-  public static final StorageLocationReport[] EMPTY_ARRAY = {};
 
   private final String id;
   private final boolean failed;
   private final long capacity;
   private final long scmUsed;
   private final long remaining;
+  private final StorageType storageType;
+  private final String storageLocation;
 
-  public StorageLocationReport(String id, boolean failed,
-      long capacity, long scmUsed, long remaining) {
+  private StorageLocationReport(String id, boolean failed, long capacity,
+      long scmUsed, long remaining, StorageType storageType,
+      String storageLocation) {
     this.id = id;
     this.failed = failed;
     this.capacity = capacity;
     this.scmUsed = scmUsed;
     this.remaining = remaining;
+    this.storageType = storageType;
+    this.storageLocation = storageLocation;
   }
 
   public String getId() {
@@ -60,4 +72,221 @@ public long getRemaining() {
     return remaining;
   }
 
+  public String getStorageLocation() {
+    return storageLocation;
+  }
+
+  public StorageType getStorageType() {
+    return storageType;
+  }
+
+
+  private StorageTypeProto getStorageTypeProto() throws
+      IOException {
+    StorageTypeProto storageTypeProto;
+    switch (getStorageType()) {
+    case SSD:
+      storageTypeProto = StorageTypeProto.SSD;
+      break;
+    case DISK:
+      storageTypeProto = StorageTypeProto.DISK;
+      break;
+    case ARCHIVE:
+      storageTypeProto = StorageTypeProto.ARCHIVE;
+      break;
+    case PROVIDED:
+      storageTypeProto = StorageTypeProto.PROVIDED;
+      break;
+    case RAM_DISK:
+      storageTypeProto = StorageTypeProto.RAM_DISK;
+      break;
+    default:
+      throw new IOException("Illegal Storage Type specified");
+    }
+    return storageTypeProto;
+  }
+
+  private static StorageType getStorageType(StorageTypeProto proto) throws
+      IOException {
+    StorageType storageType;
+    switch (proto) {
+    case SSD:
+      storageType = StorageType.SSD;
+      break;
+    case DISK:
+      storageType = StorageType.DISK;
+      break;
+    case ARCHIVE:
+      storageType = StorageType.ARCHIVE;
+      break;
+    case PROVIDED:
+      storageType = StorageType.PROVIDED;
+      break;
+    case RAM_DISK:
+      storageType = StorageType.RAM_DISK;
+      break;
+    default:
+      throw new IOException("Illegal Storage Type specified");
+    }
+    return storageType;
+  }
+
+  /**
+   * Returns the SCMStorageReport protoBuf message for the Storage Location
+   * report.
+   * @return SCMStorageReport
+   * @throws IOException In case, the storage type specified is invalid.
+   */
+  public SCMStorageReport getProtoBufMessage() throws IOException{
+    SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
+    return srb.setStorageUuid(getId())
+        .setCapacity(getCapacity())
+        .setScmUsed(getScmUsed())
+        .setRemaining(getRemaining())
+        .setStorageType(getStorageTypeProto())
+        .setStorageLocation(getStorageLocation())
+        .setFailed(isFailed())
+        .build();
+  }
+
+  /**
+   * Returns the StorageLocationReport from the protoBuf message.
+   * @param report SCMStorageReport
+   * @return StorageLocationReport
+   * @throws IOException in case of invalid storage type
+   */
+
+  public static StorageLocationReport getFromProtobuf(SCMStorageReport report)
+      throws IOException {
+    StorageLocationReport.Builder builder = StorageLocationReport.newBuilder();
+    builder.setId(report.getStorageUuid())
+        .setStorageLocation(report.getStorageLocation());
+    if (report.hasCapacity()) {
+      builder.setCapacity(report.getCapacity());
+    }
+    if (report.hasScmUsed()) {
+      builder.setScmUsed(report.getScmUsed());
+    }
+    if (report.hasStorageType()) {
+      builder.setStorageType(getStorageType(report.getStorageType()));
+    }
+    if (report.hasRemaining()) {
+      builder.setRemaining(report.getRemaining());
+    }
+
+    if (report.hasFailed()) {
+      builder.setFailed(report.getFailed());
+    }
+    return builder.build();
+  }
+
+  /**
+   * Returns StorageLocation.Builder instance.
+   *
+   * @return StorageLocation.Builder
+   */
+  public static Builder newBuilder() {
+    return new Builder();
+  }
+
+  /**
+   * Builder class for building StorageLocationReport.
+   */
+  public static class Builder {
+    private String id;
+    private boolean failed;
+    private long capacity;
+    private long scmUsed;
+    private long remaining;
+    private StorageType storageType;
+    private String storageLocation;
+
+    /**
+     * Sets the storageId.
+     *
+     * @param id storageId
+     * @return StorageLocationReport.Builder
+     */
+    public Builder setId(String id) {
+      this.id = id;
+      return this;
+    }
+
+    /**
+     * Sets whether the volume failed or not.
+     *
+     * @param failed whether volume failed or not
+     * @return StorageLocationReport.Builder
+     */
+    public Builder setFailed(boolean failed) {
+      this.failed = failed;
+      return this;
+    }
+
+    /**
+     * Sets the capacity of volume.
+     *
+     * @param capacity capacity
+     * @return StorageLocationReport.Builder
+     */
+    public Builder setCapacity(long capacity) {
+      this.capacity = capacity;
+      return this;
+    }
+    /**
+     * Sets the scmUsed Value.
+     *
+     * @param scmUsed storage space used by scm
+     * @return StorageLocationReport.Builder
+     */
+    public Builder setScmUsed(long scmUsed) {
+      this.scmUsed = scmUsed;
+      return this;
+    }
+
+    /**
+     * Sets the remaining free space value.
+     *
+     * @param remaining remaining free space
+     * @return StorageLocationReport.Builder
+     */
+    public Builder setRemaining(long remaining) {
+      this.remaining = remaining;
+      return this;
+    }
+
+    /**
+     * Sets the storageType.
+     *
+     * @param storageType type of the storage used
+     * @return StorageLocationReport.Builder
+     */
+    public Builder setStorageType(StorageType storageType) {
+      this.storageType = storageType;
+      return this;
+    }
+
+    /**
+     * Sets the storageLocation.
+     *
+     * @param storageLocation location of the volume
+     * @return StorageLocationReport.Builder
+     */
+    public Builder setStorageLocation(String storageLocation) {
+      this.storageLocation = storageLocation;
+      return this;
+    }
+
+    /**
+     * Builds and returns StorageLocationReport instance.
+     *
+     * @return StorageLocationReport
+     */
+    public StorageLocationReport build() {
+      return new StorageLocationReport(id, failed, capacity, scmUsed,
+          remaining, storageType, storageLocation);
+    }
+
+  }
+
 }

hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto

@@ -136,12 +136,25 @@ message SCMNodeReport {
   repeated SCMStorageReport storageReport = 1;
 }
 
+/**
+ * Types of recognized storage media.
+ */
+enum StorageTypeProto {
+  DISK = 1;
+  SSD = 2;
+  ARCHIVE = 3;
+  RAM_DISK = 4;
+  PROVIDED = 5;
+}
+
 message SCMStorageReport {
   required string storageUuid = 1;
-  optional uint64 capacity = 2 [default = 0];
-  optional uint64 scmUsed = 3 [default = 0];
-  optional uint64 remaining = 4 [default = 0];
-  //optional hadoop.hdfs.StorageTypeProto storageType = 5 [default = DISK];
+  required string storageLocation = 2;
+  optional uint64 capacity = 3 [default = 0];
+  optional uint64 scmUsed = 4 [default = 0];
+  optional uint64 remaining = 5 [default = 0];
+  optional StorageTypeProto storageType = 6 [default = DISK];
+  optional bool failed = 7 [default = false];
 }
 
 message SCMRegisterRequestProto {