From 979c9ca2ca89e99dc7165abfa29c78d66de43d9a Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 15 Jul 2015 04:41:54 +0900 Subject: [PATCH] HDFS-8742. Inotify: Support event for OP_TRUNCATE. Contributed by Surendra Singh Lilhore. --- .../org/apache/hadoop/hdfs/inotify/Event.java | 37 ++++++++++++++++++- .../src/main/proto/inotify.proto | 7 ++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/protocolPB/PBHelper.java | 17 +++++++++ .../InotifyFSEditLogOpTranslator.java | 4 ++ .../hdfs/TestDFSInotifyEventInputStream.java | 17 ++++++++- 6 files changed, 83 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java index 53eefa0425..dee17a9192 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java @@ -35,7 +35,7 @@ @InterfaceStability.Unstable public abstract class Event { public static enum EventType { - CREATE, CLOSE, APPEND, RENAME, METADATA, UNLINK + CREATE, CLOSE, APPEND, RENAME, METADATA, UNLINK, TRUNCATE } private EventType eventType; @@ -542,4 +542,39 @@ public long getTimestamp() { return timestamp; } } + + /** + * Sent when a file is truncated. + */ + public static class TruncateEvent extends Event { + private String path; + private long fileSize; + private long timestamp; + + + public TruncateEvent(String path, long fileSize, long timestamp) { + super(EventType.TRUNCATE); + this.path = path; + this.fileSize = fileSize; + this.timestamp = timestamp; + } + + public String getPath() { + return path; + } + + /** + * The size of the truncated file in bytes. + */ + public long getFileSize() { + return fileSize; + } + + /** + * The time when this event occurred, in milliseconds since the epoch. + */ + public long getTimestamp() { + return timestamp; + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto index 5b78fe62de..5339902958 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto @@ -41,6 +41,7 @@ enum EventType { EVENT_RENAME = 0x3; EVENT_METADATA = 0x4; EVENT_UNLINK = 0x5; + EVENT_TRUNCATE = 0x6; } message EventProto { @@ -87,6 +88,12 @@ message CloseEventProto { required int64 timestamp = 3; } +message TruncateEventProto { + required string path = 1; + required int64 fileSize = 2; + required int64 timestamp = 3; +} + message AppendEventProto { required string path = 1; optional bool newBlock = 2 [default = false]; diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 14f3403708..20bdef0cd1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -719,6 +719,9 @@ Release 2.8.0 - UNRELEASED HDFS-8541. Mover should exit with NO_MOVE_PROGRESS if there is no move progress. (Surendra Singh Lilhore via szetszwo) + HDFS-8742. Inotify: Support event for OP_TRUNCATE. + (Surendra Singh Lilhore via aajisaka) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 32d961477e..4ca5b26243 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -2685,6 +2685,12 @@ public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws .timestamp(unlink.getTimestamp()) .build()); break; + case EVENT_TRUNCATE: + InotifyProtos.TruncateEventProto truncate = + InotifyProtos.TruncateEventProto.parseFrom(p.getContents()); + events.add(new Event.TruncateEvent(truncate.getPath(), + truncate.getFileSize(), truncate.getTimestamp())); + break; default: throw new RuntimeException("Unexpected inotify event type: " + p.getType()); @@ -2791,6 +2797,17 @@ public static GetEditsFromTxidResponseProto convertEditsResponse(EventBatchList .setTimestamp(ue.getTimestamp()).build().toByteString() ).build()); break; + case TRUNCATE: + Event.TruncateEvent te = (Event.TruncateEvent) e; + events.add(InotifyProtos.EventProto.newBuilder() + .setType(InotifyProtos.EventType.EVENT_TRUNCATE) + .setContents( + InotifyProtos.TruncateEventProto.newBuilder() + .setPath(te.getPath()) + .setFileSize(te.getFileSize()) + .setTimestamp(te.getTimestamp()).build().toByteString() + ).build()); + break; default: throw new RuntimeException("Unexpected inotify event: " + e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java index 5345b46de5..0918107464 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java @@ -176,6 +176,10 @@ public static EventBatch translate(FSEditLogOp op) { .metadataType(Event.MetadataUpdateEvent.MetadataType.ACLS) .path(saOp.src) .acls(saOp.aclEntries).build() }); + case OP_TRUNCATE: + FSEditLogOp.TruncateOp tOp = (FSEditLogOp.TruncateOp) op; + return new EventBatch(op.txid, new Event[] { + new Event.TruncateEvent(tOp.src, tOp.newLength, tOp.timestamp) }); default: return null; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java index ba33bd3ec4..385d653478 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java @@ -102,6 +102,8 @@ public void testBasic() throws IOException, URISyntaxException, DFSTestUtil.createFile(fs, new Path("/file"), BLOCK_SIZE, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path("/file3"), BLOCK_SIZE, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path("/file5"), BLOCK_SIZE, (short) 1, 0L); + DFSTestUtil.createFile(fs, new Path("/truncate_file"), + BLOCK_SIZE * 2, (short) 1, 0L); DFSInotifyEventInputStream eis = client.getInotifyEventStream(); client.rename("/file", "/file4", null); // RenameOp -> RenameEvent client.rename("/file4", "/file2"); // RenameOldOp -> RenameEvent @@ -136,7 +138,8 @@ public void testBasic() throws IOException, URISyntaxException, "user::rwx,user:foo:rw-,group::r--,other::---", true)); client.removeAcl("/file5"); // SetAclOp -> MetadataUpdateEvent client.rename("/file5", "/dir"); // RenameOldOp -> RenameEvent - + //TruncateOp -> TruncateEvent + client.truncate("/truncate_file", BLOCK_SIZE); EventBatch batch = null; // RenameOp @@ -354,6 +357,18 @@ public void testBasic() throws IOException, URISyntaxException, Assert.assertTrue(re3.getSrcPath().equals("/file5")); Assert.assertTrue(re.getTimestamp() > 0); + // TruncateOp + batch = waitForNextEvents(eis); + Assert.assertEquals(1, batch.getEvents().length); + txid = checkTxid(batch, txid); + Assert + .assertTrue(batch.getEvents()[0].getEventType() == + Event.EventType.TRUNCATE); + Event.TruncateEvent et = ((Event.TruncateEvent) batch.getEvents()[0]); + Assert.assertTrue(et.getPath().equals("/truncate_file")); + Assert.assertTrue(et.getFileSize() == BLOCK_SIZE); + Assert.assertTrue(et.getTimestamp() > 0); + // Returns null when there are no further events Assert.assertTrue(eis.poll() == null);