HDFS-7513. HDFS inotify: add defaultBlockSize to CreateEvent (cmccabe)

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -457,6 +457,8 @@ Release 2.7.0 - UNRELEASED
 
     HDFS-7426. Change nntop JMX format to be a JSON blob. (wang)
 
+    HDFS-7513. HDFS inotify: add defaultBlockSize to CreateEvent (cmccabe)
+
   OPTIMIZATIONS
 
     HDFS-7454. Reduce memory footprint for AclEntries in NameNode.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java

@@ -101,6 +101,7 @@ public static enum INodeType {
     private FsPermission perms;
     private String symlinkTarget;
     private boolean overwrite;
+    private long defaultBlockSize;
 
     public static class Builder {
       private INodeType iNodeType;
@@ -112,6 +113,7 @@ public static class Builder {
       private FsPermission perms;
       private String symlinkTarget;
       private boolean overwrite;
+      private long defaultBlockSize = 0;
 
       public Builder iNodeType(INodeType type) {
         this.iNodeType = type;
@@ -158,6 +160,11 @@ public Builder overwrite(boolean overwrite) {
         return this;
       }
 
+      public Builder defaultBlockSize(long defaultBlockSize) {
+        this.defaultBlockSize = defaultBlockSize;
+        return this;
+      }
+
       public CreateEvent build() {
         return new CreateEvent(this);
       }
@@ -174,6 +181,7 @@ private CreateEvent(Builder b) {
       this.perms = b.perms;
       this.symlinkTarget = b.symlinkTarget;
       this.overwrite = b.overwrite;
+      this.defaultBlockSize = b.defaultBlockSize;
     }
 
     public INodeType getiNodeType() {
@@ -220,6 +228,10 @@ public String getSymlinkTarget() {
     public boolean getOverwrite() {
       return overwrite;
     }
+
+    public long getDefaultBlockSize() {
+      return defaultBlockSize;
+    }
   }
 
   /**
@@ -398,11 +410,36 @@ public static class RenameEvent extends Event {
     private String dstPath;
     private long timestamp;
 
-    public RenameEvent(String srcPath, String dstPath, long timestamp) {
+    public static class Builder {
+      private String srcPath;
+      private String dstPath;
+      private long timestamp;
+
+      public Builder srcPath(String srcPath) {
+        this.srcPath = srcPath;
+        return this;
+      }
+
+      public Builder dstPath(String dstPath) {
+        this.dstPath = dstPath;
+        return this;
+      }
+
+      public Builder timestamp(long timestamp) {
+        this.timestamp = timestamp;
+        return this;
+      }
+
+      public RenameEvent build() {
+        return new RenameEvent(this);
+      }
+    }
+
+    private RenameEvent(Builder builder) {
       super(EventType.RENAME);
-      this.srcPath = srcPath;
+      this.srcPath = builder.srcPath;
-      this.dstPath = dstPath;
+      this.dstPath = builder.dstPath;
-      this.timestamp = timestamp;
+      this.timestamp = builder.timestamp;
     }
 
     public String getSrcPath() {
@@ -427,9 +464,22 @@ public long getTimestamp() {
   public static class AppendEvent extends Event {
     private String path;
 
-    public AppendEvent(String path) {
+    public static class Builder {
+      private String path;
+
+      public Builder path(String path) {
+        this.path = path;
+        return this;
+      }
+
+      public AppendEvent build() {
+        return new AppendEvent(this);
+      }
+    }
+
+    private AppendEvent(Builder b) {
       super(EventType.APPEND);
-      this.path = path;
+      this.path = b.path;
     }
 
     public String getPath() {
@@ -444,10 +494,29 @@ public static class UnlinkEvent extends Event {
     private String path;
     private long timestamp;
 
-    public UnlinkEvent(String path, long timestamp) {
+    public static class Builder {
+      private String path;
+      private long timestamp;
+
+      public Builder path(String path) {
+        this.path = path;
+        return this;
+      }
+
+      public Builder timestamp(long timestamp) {
+        this.timestamp = timestamp;
+        return this;
+      }
+
+      public UnlinkEvent build() {
+        return new UnlinkEvent(this);
+      }
+    }
+
+    private UnlinkEvent(Builder builder) {
       super(EventType.UNLINK);
-      this.path = path;
+      this.path = builder.path;
-      this.timestamp = timestamp;
+      this.timestamp = builder.timestamp;
     }
 
     public String getPath() {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java

@@ -2566,6 +2566,7 @@ public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
                 .replication(create.getReplication())
                 .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
                     create.getSymlinkTarget())
+                .defaultBlockSize(create.getDefaultBlockSize())
                 .overwrite(create.getOverwrite()).build());
             break;
           case EVENT_METADATA:
@@ -2592,19 +2593,26 @@ public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
           case EVENT_RENAME:
             InotifyProtos.RenameEventProto rename =
                 InotifyProtos.RenameEventProto.parseFrom(p.getContents());
-            events.add(new Event.RenameEvent(rename.getSrcPath(),
+            events.add(new Event.RenameEvent.Builder()
-                rename.getDestPath(), rename.getTimestamp()));
+                  .srcPath(rename.getSrcPath())
+                  .dstPath(rename.getDestPath())
+                  .timestamp(rename.getTimestamp())
+                  .build());
             break;
           case EVENT_APPEND:
             InotifyProtos.AppendEventProto reopen =
                 InotifyProtos.AppendEventProto.parseFrom(p.getContents());
-            events.add(new Event.AppendEvent(reopen.getPath()));
+            events.add(new Event.AppendEvent.Builder()
+                  .path(reopen.getPath())
+                  .build());
             break;
           case EVENT_UNLINK:
             InotifyProtos.UnlinkEventProto unlink =
                 InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
-            events.add(new Event.UnlinkEvent(unlink.getPath(),
+            events.add(new Event.UnlinkEvent.Builder()
-                unlink.getTimestamp()));
+                  .path(unlink.getPath())
+                  .timestamp(unlink.getTimestamp())
+                  .build());
             break;
           default:
             throw new RuntimeException("Unexpected inotify event type: " +
@@ -2650,6 +2658,7 @@ public static GetEditsFromTxidResponseProto convertEditsResponse(EventBatchList
                         .setReplication(ce2.getReplication())
                         .setSymlinkTarget(ce2.getSymlinkTarget() == null ?
                             "" : ce2.getSymlinkTarget())
+                        .setDefaultBlockSize(ce2.getDefaultBlockSize())
                         .setOverwrite(ce2.getOverwrite()).build().toByteString()
                 ).build());
             break;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java

@@ -53,10 +53,13 @@ public static EventBatch translate(FSEditLogOp op) {
             .groupName(addOp.permissions.getGroupName())
             .perms(addOp.permissions.getPermission())
             .overwrite(addOp.overwrite)
+            .defaultBlockSize(addOp.blockSize)
             .iNodeType(Event.CreateEvent.INodeType.FILE).build() });
-      } else {
+      } else { // append
         return new EventBatch(op.txid,
-            new Event[] { new Event.AppendEvent(addOp.path) });
+            new Event[]{new Event.AppendEvent.Builder()
+                .path(addOp.path)
+                .build()});
       }
     case OP_CLOSE:
       FSEditLogOp.CloseOp cOp = (FSEditLogOp.CloseOp) op;
@@ -72,25 +75,40 @@ public static EventBatch translate(FSEditLogOp op) {
     case OP_CONCAT_DELETE:
       FSEditLogOp.ConcatDeleteOp cdOp = (FSEditLogOp.ConcatDeleteOp) op;
       List<Event> events = Lists.newArrayList();
-      events.add(new Event.AppendEvent(cdOp.trg));
+      events.add(new Event.AppendEvent.Builder()
+          .path(cdOp.trg)
+          .build());
       for (String src : cdOp.srcs) {
-        events.add(new Event.UnlinkEvent(src, cdOp.timestamp));
+        events.add(new Event.UnlinkEvent.Builder()
+          .path(src)
+          .timestamp(cdOp.timestamp)
+          .build());
       }
       events.add(new Event.CloseEvent(cdOp.trg, -1, cdOp.timestamp));
       return new EventBatch(op.txid, events.toArray(new Event[0]));
     case OP_RENAME_OLD:
       FSEditLogOp.RenameOldOp rnOpOld = (FSEditLogOp.RenameOldOp) op;
       return new EventBatch(op.txid, new Event[] {
-          new Event.RenameEvent(rnOpOld.src,
+          new Event.RenameEvent.Builder()
-              rnOpOld.dst, rnOpOld.timestamp) });
+              .srcPath(rnOpOld.src)
+              .dstPath(rnOpOld.dst)
+              .timestamp(rnOpOld.timestamp)
+              .build() });
     case OP_RENAME:
       FSEditLogOp.RenameOp rnOp = (FSEditLogOp.RenameOp) op;
       return new EventBatch(op.txid, new Event[] {
-          new Event.RenameEvent(rnOp.src, rnOp.dst, rnOp.timestamp) });
+          new Event.RenameEvent.Builder()
+            .srcPath(rnOp.src)
+            .dstPath(rnOp.dst)
+            .timestamp(rnOp.timestamp)
+            .build() });
     case OP_DELETE:
       FSEditLogOp.DeleteOp delOp = (FSEditLogOp.DeleteOp) op;
       return new EventBatch(op.txid, new Event[] {
-          new Event.UnlinkEvent(delOp.path, delOp.timestamp) });
+          new Event.UnlinkEvent.Builder()
+            .path(delOp.path)
+            .timestamp(delOp.timestamp)
+            .build() });
     case OP_MKDIR:
       FSEditLogOp.MkdirOp mkOp = (FSEditLogOp.MkdirOp) op;
       return new EventBatch(op.txid,

hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto

@@ -78,6 +78,7 @@ message CreateEventProto {
   optional int32 replication = 7;
   optional string symlinkTarget = 8;
   optional bool overwrite = 9;
+  optional int64 defaultBlockSize = 10 [default=0];
 }
 
 message CloseEventProto {

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java

@@ -170,6 +170,7 @@ public void testBasic() throws IOException, URISyntaxException,
       Assert.assertTrue(ce.getReplication() > 0);
       Assert.assertTrue(ce.getSymlinkTarget() == null);
       Assert.assertTrue(ce.getOverwrite());
+      Assert.assertEquals(BLOCK_SIZE, ce.getDefaultBlockSize());
 
       // CloseOp
       batch = waitForNextEvents(eis);
@@ -186,7 +187,8 @@ public void testBasic() throws IOException, URISyntaxException,
       Assert.assertEquals(1, batch.getEvents().length);
       txid = checkTxid(batch, txid);
       Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.APPEND);
-      Assert.assertTrue(((Event.AppendEvent) batch.getEvents()[0]).getPath().equals("/file2"));
+      Event.AppendEvent append2 = (Event.AppendEvent)batch.getEvents()[0];
+      Assert.assertEquals("/file2", append2.getPath());
 
       // CloseOp
       batch = waitForNextEvents(eis);